From 02fe95c5c4d8b4980b03719e10d53f98d2c0f59d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 21 Oct 2022 14:39:21 -0700 Subject: [PATCH 01/15] Add sccache to manywheel binary build --- common/install_cache.sh | 121 ++++++++++++++++++++++++++++++++++++++ manywheel/Dockerfile | 6 ++ manywheel/build_common.sh | 52 ++++++++++++++++ 3 files changed, 179 insertions(+) create mode 100644 common/install_cache.sh diff --git a/common/install_cache.sh b/common/install_cache.sh new file mode 100644 index 000000000..6b48f11e4 --- /dev/null +++ b/common/install_cache.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +set -ex + +install_ubuntu() { + echo "Preparing to build sccache from source" + apt-get update + # libssl-dev will not work as it is upgraded to libssl3 in Ubuntu-22.04. + # Instead use lib and headers from OpenSSL1.1 installed in `install_openssl.sh`` + apt-get install -y cargo + echo "Checking out sccache repo" + git clone https://github.com/pytorch/sccache + cd sccache + echo "Building sccache" + cargo build --release + cp target/release/sccache /opt/cache/bin + echo "Cleaning up" + cd .. + rm -rf sccache + apt-get remove -y cargo rustc + apt-get autoclean && apt-get clean +} + +install_binary() { + echo "Downloading sccache binary from S3 repo" + curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache +} + +mkdir -p /opt/cache/bin +mkdir -p /opt/cache/lib +sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment +export PATH="/opt/cache/bin:$PATH" + +# Setup compiler cache +if [ -n "$ROCM_VERSION" ]; then + curl --retry 3 http://repo.radeon.com/misc/.sccache_amd/sccache -o /opt/cache/bin/sccache +else + ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') + case "$ID" in + ubuntu) + install_ubuntu + ;; + *) + install_binary + ;; + esac +fi +chmod a+x /opt/cache/bin/sccache + +function write_sccache_stub() { + # Unset LD_PRELOAD for ps because of asan + ps issues + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90589 + printf "#!/bin/sh\nif [ \$(env -u LD_PRELOAD ps -p \$PPID -o comm=) != sccache ]; then\n exec sccache $(which $1) \"\$@\"\nelse\n exec $(which $1) \"\$@\"\nfi" > "/opt/cache/bin/$1" + chmod a+x "/opt/cache/bin/$1" +} + +write_sccache_stub cc +write_sccache_stub c++ +write_sccache_stub gcc +write_sccache_stub g++ + +# NOTE: See specific ROCM_VERSION case below. +if [ "x$ROCM_VERSION" = x ]; then + write_sccache_stub clang + write_sccache_stub clang++ +fi + +if [ -n "$CUDA_VERSION" ]; then + # TODO: This is a workaround for the fact that PyTorch's FindCUDA + # implementation cannot find nvcc if it is setup this way, because it + # appears to search for the nvcc in PATH, and use its path to infer + # where CUDA is installed. Instead, we install an nvcc symlink outside + # of the PATH, and set CUDA_NVCC_EXECUTABLE so that we make use of it. + + write_sccache_stub nvcc + mv /opt/cache/bin/nvcc /opt/cache/lib/ +fi + +if [ -n "$ROCM_VERSION" ]; then + # ROCm compiler is hcc or clang. However, it is commonly invoked via hipcc wrapper. + # hipcc will call either hcc or clang using an absolute path starting with /opt/rocm, + # causing the /opt/cache/bin to be skipped. We must create the sccache wrappers + # directly under /opt/rocm while also preserving the original compiler names. + # Note symlinks will chain as follows: [hcc or clang++] -> clang -> clang-?? + # Final link in symlink chain must point back to original directory. + + # Original compiler is moved one directory deeper. Wrapper replaces it. + function write_sccache_stub_rocm() { + OLDCOMP=$1 + COMPNAME=$(basename $OLDCOMP) + TOPDIR=$(dirname $OLDCOMP) + WRAPPED="$TOPDIR/original/$COMPNAME" + mv "$OLDCOMP" "$WRAPPED" + printf "#!/bin/sh\nexec sccache $WRAPPED \"\$@\"" > "$OLDCOMP" + chmod a+x "$OLDCOMP" + } + + if [[ -e "/opt/rocm/hcc/bin/hcc" ]]; then + # ROCm 3.3 or earlier. + mkdir /opt/rocm/hcc/bin/original + write_sccache_stub_rocm /opt/rocm/hcc/bin/hcc + write_sccache_stub_rocm /opt/rocm/hcc/bin/clang + write_sccache_stub_rocm /opt/rocm/hcc/bin/clang++ + # Fix last link in symlink chain, clang points to versioned clang in prior dir + pushd /opt/rocm/hcc/bin/original + ln -s ../$(readlink clang) + popd + elif [[ -e "/opt/rocm/llvm/bin/clang" ]]; then + # ROCm 3.5 and beyond. + mkdir /opt/rocm/llvm/bin/original + write_sccache_stub_rocm /opt/rocm/llvm/bin/clang + write_sccache_stub_rocm /opt/rocm/llvm/bin/clang++ + # Fix last link in symlink chain, clang points to versioned clang in prior dir + pushd /opt/rocm/llvm/bin/original + ln -s ../$(readlink clang) + popd + else + echo "Cannot find ROCm compiler." + exit 1 + fi +fi diff --git a/manywheel/Dockerfile b/manywheel/Dockerfile index 8ddee1dab..75b2b7fee 100644 --- a/manywheel/Dockerfile +++ b/manywheel/Dockerfile @@ -173,3 +173,9 @@ RUN yum install -y cmake3 && \ ln -s /usr/bin/cmake3 /usr/bin/cmake ADD ./common/install_miopen.sh install_miopen.sh RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh + +# Install compiler cache (do this last, so we get priority in PATH) +COPY ./common/install_cache.sh install_cache.sh +ENV PATH /opt/cache/bin:$PATH +RUN bash ./install_cache.sh && rm install_cache.sh +ENV CMAKE_CUDA_COMPILER_LAUNCHER=/opt/cache/bin/sccache diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh index c213145ef..6336f93d3 100644 --- a/manywheel/build_common.sh +++ b/manywheel/build_common.sh @@ -4,6 +4,58 @@ set -ex SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +# Courtesy of pytorch/.jenkins/pytorch/common_utils.sh +# +# - 1st arg: code to add +# - remaining args: names of traps to modify +# +trap_add() { + trap_add_cmd=$1; shift || fatal "${FUNCNAME[0]} usage error" + for trap_add_name in "$@"; do + trap -- "$( + # helper fn to get existing trap command from output + # of trap -p + extract_trap_cmd() { printf '%s\n' "$3"; } + # print existing trap command with newline + eval "extract_trap_cmd $(trap -p "${trap_add_name}")" + # print the new trap command + printf '%s\n' "${trap_add_cmd}" + )" "${trap_add_name}" \ + || fatal "unable to add to trap ${trap_add_name}" + done +} +# set the trace attribute for the above function. this is +# required to modify DEBUG or RETURN traps because functions don't +# inherit them unless the trace attribute is set +declare -f -t trap_add + +# Initialize sccache +if which sccache > /dev/null; then + # Save sccache logs to file + sccache --stop-server > /dev/null 2>&1 || true + rm -f ~/sccache_error.log || true + + export SCCACHE_IDLE_TIMEOUT=1200 + export SCCACHE_ERROR_LOG=~/sccache_error.log + export RUST_LOG=sccache::server=error + + # Report sccache stats for easier debugging + sccache --zero-stats + + # Report sccache stats for easier debugging + sccache --zero-stats + function sccache_epilogue() { + echo "::group::Sccache Compilation Log" + echo '=================== sccache compilation log ===================' + python "$script_dir/print_sccache_log.py" ~/sccache_error.log 2>/dev/null + echo '=========== If your build fails, please take a look at the log above for possible reasons ===========' + sccache --show-stats + sccache --stop-server || true + echo "::endgroup::" + } + + trap_add sccache_epilogue EXIT +fi # Require only one python installation if [[ -z "$DESIRED_PYTHON" ]]; then From e7608179efd287af102e40941fc24abff8d8a5bd Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 21 Oct 2022 14:53:12 -0700 Subject: [PATCH 02/15] Handle rocm and cpu and cuda --- manywheel/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/manywheel/Dockerfile b/manywheel/Dockerfile index 75b2b7fee..17b3fd887 100644 --- a/manywheel/Dockerfile +++ b/manywheel/Dockerfile @@ -150,6 +150,12 @@ RUN yum install -y cmake3 && \ RUN yum install -y http://repo.okay.com.mx/centos/7/x86_64/release/okay-release-1-5.el7.noarch.rpm RUN yum install -y ninja-build +# Install compiler cache (do this last, so we get priority in PATH) +COPY ./common/install_cache.sh install_cache.sh +ENV PATH /opt/cache/bin:$PATH +RUN bash ./install_cache.sh && rm install_cache.sh +ENV CMAKE_CUDA_COMPILER_LAUNCHER=/opt/cache/bin/sccache + FROM cpu_final as cuda_final RUN rm -rf /usr/local/cuda-${BASE_CUDA_VERSION} COPY --from=cuda /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION} @@ -177,5 +183,5 @@ RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh # Install compiler cache (do this last, so we get priority in PATH) COPY ./common/install_cache.sh install_cache.sh ENV PATH /opt/cache/bin:$PATH -RUN bash ./install_cache.sh && rm install_cache.sh +RUN ROCM_VERSION=${ROCM_VERSION} bash ./install_cache.sh && rm install_cache.sh ENV CMAKE_CUDA_COMPILER_LAUNCHER=/opt/cache/bin/sccache From cf7fdd025c2350723537047cf39da154a9b39034 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 21 Oct 2022 16:35:07 -0700 Subject: [PATCH 03/15] Fix copy paste duplication --- manywheel/build_common.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh index 6336f93d3..3dd4ccb42 100644 --- a/manywheel/build_common.sh +++ b/manywheel/build_common.sh @@ -39,19 +39,11 @@ if which sccache > /dev/null; then export SCCACHE_ERROR_LOG=~/sccache_error.log export RUST_LOG=sccache::server=error - # Report sccache stats for easier debugging - sccache --zero-stats - # Report sccache stats for easier debugging sccache --zero-stats function sccache_epilogue() { - echo "::group::Sccache Compilation Log" - echo '=================== sccache compilation log ===================' - python "$script_dir/print_sccache_log.py" ~/sccache_error.log 2>/dev/null - echo '=========== If your build fails, please take a look at the log above for possible reasons ===========' sccache --show-stats sccache --stop-server || true - echo "::endgroup::" } trap_add sccache_epilogue EXIT From 09fcfc01061645cb7b448cb674a11d25db9abebc Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Oct 2022 14:24:15 -0700 Subject: [PATCH 04/15] Debug with the new docker cuda images --- .github/workflows/build-manywheel-images.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-manywheel-images.yml b/.github/workflows/build-manywheel-images.yml index b62507bbd..0cb06991c 100644 --- a/.github/workflows/build-manywheel-images.yml +++ b/.github/workflows/build-manywheel-images.yml @@ -37,9 +37,10 @@ jobs: uses: actions/checkout@v2 - name: Authenticate if WITH_PUSH run: | - if [[ "${WITH_PUSH}" == true ]]; then - echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin - fi + # DEBUG with new cuda images + #if [[ "${WITH_PUSH}" == true ]]; then + echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin + # fi - name: Build Docker Image run: | manywheel/build_docker.sh From 7fe191b05480c077ea5ecee2118d02f2422fc819 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Oct 2022 14:46:09 -0700 Subject: [PATCH 05/15] Uncomment debug --- .github/workflows/build-manywheel-images.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-manywheel-images.yml b/.github/workflows/build-manywheel-images.yml index 0cb06991c..b62507bbd 100644 --- a/.github/workflows/build-manywheel-images.yml +++ b/.github/workflows/build-manywheel-images.yml @@ -37,10 +37,9 @@ jobs: uses: actions/checkout@v2 - name: Authenticate if WITH_PUSH run: | - # DEBUG with new cuda images - #if [[ "${WITH_PUSH}" == true ]]; then - echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin - # fi + if [[ "${WITH_PUSH}" == true ]]; then + echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin + fi - name: Build Docker Image run: | manywheel/build_docker.sh From 37bd8cc2361699c42605a293c893e8863cf947cb Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Oct 2022 15:34:36 -0700 Subject: [PATCH 06/15] Testing add-sccache-support --- .github/workflows/build-manywheel-images.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-manywheel-images.yml b/.github/workflows/build-manywheel-images.yml index b62507bbd..2a56f18d8 100644 --- a/.github/workflows/build-manywheel-images.yml +++ b/.github/workflows/build-manywheel-images.yml @@ -3,7 +3,8 @@ name: Build manywheel docker images on: push: branches: - main + - main + - add-sccache-support paths: - .github/workflows/build-manywheel-images.yml - manywheel/Dockerfile From 5934f8b8bc0f8a2d4e13fad9ef76cefbe6e2f7d9 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Oct 2022 15:39:25 -0700 Subject: [PATCH 07/15] Remove reference to the debug branch --- .github/workflows/build-manywheel-images.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-manywheel-images.yml b/.github/workflows/build-manywheel-images.yml index 2a56f18d8..1c5707b9c 100644 --- a/.github/workflows/build-manywheel-images.yml +++ b/.github/workflows/build-manywheel-images.yml @@ -4,7 +4,6 @@ on: push: branches: - main - - add-sccache-support paths: - .github/workflows/build-manywheel-images.yml - manywheel/Dockerfile From 0de0b2866e5312c7b4b55abcb2a11f72962e2ab3 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Oct 2022 20:18:40 -0700 Subject: [PATCH 08/15] Install sccache from source --- common/install_cache.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/common/install_cache.sh b/common/install_cache.sh index 6b48f11e4..6bf603f5c 100644 --- a/common/install_cache.sh +++ b/common/install_cache.sh @@ -21,6 +21,22 @@ install_ubuntu() { apt-get autoclean && apt-get clean } +install_centos() { + # Install sccache from PyTorch fork to get the version that supports NVCC + echo "Preparing to build sccache from source" + yum install -y cargo openssl-devel + echo "Checking out sccache repo" + git clone https://github.com/pytorch/sccache + cd sccache + echo "Building sccache" + cargo build --release + cp target/release/sccache /opt/cache/bin + echo "Cleaning up" + cd .. + rm -rf sccache + yum remove -y cargo rustc +} + install_binary() { echo "Downloading sccache binary from S3 repo" curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache @@ -40,6 +56,9 @@ else ubuntu) install_ubuntu ;; + centos) + install_centos + ;; *) install_binary ;; From c58b2ff7fa1a661ebe3b7289d40bded83f2b404e Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Oct 2022 23:26:37 -0700 Subject: [PATCH 09/15] PyTorch fork of sccache does not work with centos --- common/install_cache.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/install_cache.sh b/common/install_cache.sh index 6bf603f5c..92bcf25a7 100644 --- a/common/install_cache.sh +++ b/common/install_cache.sh @@ -22,11 +22,11 @@ install_ubuntu() { } install_centos() { - # Install sccache from PyTorch fork to get the version that supports NVCC + # Install sccache from source get the version that supports NVCC echo "Preparing to build sccache from source" yum install -y cargo openssl-devel echo "Checking out sccache repo" - git clone https://github.com/pytorch/sccache + git clone https://github.com/mozilla/sccache cd sccache echo "Building sccache" cargo build --release From ef33b03fffcda91590c7454f69844c896027c6bb Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Oct 2022 23:37:27 -0700 Subject: [PATCH 10/15] Not using sccache if it is not setup properly --- manywheel/build_common.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh index 3dd4ccb42..c0801e07a 100644 --- a/manywheel/build_common.sh +++ b/manywheel/build_common.sh @@ -30,7 +30,7 @@ trap_add() { declare -f -t trap_add # Initialize sccache -if which sccache > /dev/null; then +if [[ -n "$SCCACHE_BUCKET" ]] && which sccache > /dev/null; then # Save sccache logs to file sccache --stop-server > /dev/null 2>&1 || true rm -f ~/sccache_error.log || true @@ -47,6 +47,14 @@ if which sccache > /dev/null; then } trap_add sccache_epilogue EXIT +else + # Not using sscache if it's not setup properly + sudo rm -f /opt/cache/bin/cc + sudo rm -f /opt/cache/bin/c++ + sudo rm -f /opt/cache/bin/clang + sudo rm -f /opt/cache/bin/clang++ + sudo rm -f /opt/cache/bin/gcc + sudo rm -f /opt/cache/bin/g++ fi # Require only one python installation From 10ea871ae10aa3f7fa4c7ce8a51feeb594b83d12 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 25 Oct 2022 00:29:25 -0700 Subject: [PATCH 11/15] Unset sccache if it is not used --- manywheel/build_common.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh index c0801e07a..1e2044c91 100644 --- a/manywheel/build_common.sh +++ b/manywheel/build_common.sh @@ -49,12 +49,14 @@ if [[ -n "$SCCACHE_BUCKET" ]] && which sccache > /dev/null; then trap_add sccache_epilogue EXIT else # Not using sscache if it's not setup properly - sudo rm -f /opt/cache/bin/cc - sudo rm -f /opt/cache/bin/c++ - sudo rm -f /opt/cache/bin/clang - sudo rm -f /opt/cache/bin/clang++ - sudo rm -f /opt/cache/bin/gcc - sudo rm -f /opt/cache/bin/g++ + rm -f /opt/cache/bin/cc + rm -f /opt/cache/bin/c++ + rm -f /opt/cache/bin/clang + rm -f /opt/cache/bin/clang++ + rm -f /opt/cache/bin/gcc + rm -f /opt/cache/bin/g++ + + unset CMAKE_CUDA_COMPILER_LAUNCHER fi # Require only one python installation From 46923dba5d8d0d6aa6f72d28a217716ed62dc8fe Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 25 Oct 2022 00:30:44 -0700 Subject: [PATCH 12/15] Quick fix to not use sccache if it's not setup properly --- manywheel/build_common.sh | 54 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh index c213145ef..1e2044c91 100644 --- a/manywheel/build_common.sh +++ b/manywheel/build_common.sh @@ -4,6 +4,60 @@ set -ex SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +# Courtesy of pytorch/.jenkins/pytorch/common_utils.sh +# +# - 1st arg: code to add +# - remaining args: names of traps to modify +# +trap_add() { + trap_add_cmd=$1; shift || fatal "${FUNCNAME[0]} usage error" + for trap_add_name in "$@"; do + trap -- "$( + # helper fn to get existing trap command from output + # of trap -p + extract_trap_cmd() { printf '%s\n' "$3"; } + # print existing trap command with newline + eval "extract_trap_cmd $(trap -p "${trap_add_name}")" + # print the new trap command + printf '%s\n' "${trap_add_cmd}" + )" "${trap_add_name}" \ + || fatal "unable to add to trap ${trap_add_name}" + done +} +# set the trace attribute for the above function. this is +# required to modify DEBUG or RETURN traps because functions don't +# inherit them unless the trace attribute is set +declare -f -t trap_add + +# Initialize sccache +if [[ -n "$SCCACHE_BUCKET" ]] && which sccache > /dev/null; then + # Save sccache logs to file + sccache --stop-server > /dev/null 2>&1 || true + rm -f ~/sccache_error.log || true + + export SCCACHE_IDLE_TIMEOUT=1200 + export SCCACHE_ERROR_LOG=~/sccache_error.log + export RUST_LOG=sccache::server=error + + # Report sccache stats for easier debugging + sccache --zero-stats + function sccache_epilogue() { + sccache --show-stats + sccache --stop-server || true + } + + trap_add sccache_epilogue EXIT +else + # Not using sscache if it's not setup properly + rm -f /opt/cache/bin/cc + rm -f /opt/cache/bin/c++ + rm -f /opt/cache/bin/clang + rm -f /opt/cache/bin/clang++ + rm -f /opt/cache/bin/gcc + rm -f /opt/cache/bin/g++ + + unset CMAKE_CUDA_COMPILER_LAUNCHER +fi # Require only one python installation if [[ -z "$DESIRED_PYTHON" ]]; then From 2babaea0ee994369660001b15cf0aefe330a4a02 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 25 Oct 2022 00:44:59 -0700 Subject: [PATCH 13/15] Add a comment about pytorch/sccache issue on CentOS --- common/install_cache.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/install_cache.sh b/common/install_cache.sh index 92bcf25a7..a8ab2ec85 100644 --- a/common/install_cache.sh +++ b/common/install_cache.sh @@ -26,6 +26,8 @@ install_centos() { echo "Preparing to build sccache from source" yum install -y cargo openssl-devel echo "Checking out sccache repo" + # TODO: https://github.com/pytorch/sccache is very outdated, so let's take + # a note here to update it later with the latest code from upstream git clone https://github.com/mozilla/sccache cd sccache echo "Building sccache" From bed12b4145589b5db825e892fcb2ca6e4a5d87e0 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 25 Oct 2022 01:07:17 -0700 Subject: [PATCH 14/15] Pin sccache version to 0.3.0 --- common/install_cache.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/common/install_cache.sh b/common/install_cache.sh index a8ab2ec85..3da5c86c8 100644 --- a/common/install_cache.sh +++ b/common/install_cache.sh @@ -9,6 +9,8 @@ install_ubuntu() { # Instead use lib and headers from OpenSSL1.1 installed in `install_openssl.sh`` apt-get install -y cargo echo "Checking out sccache repo" + # TODO: https://github.com/pytorch/sccache is very outdated, so let's take + # a note here to update it later with the latest code from upstream git clone https://github.com/pytorch/sccache cd sccache echo "Building sccache" @@ -22,17 +24,19 @@ install_ubuntu() { } install_centos() { - # Install sccache from source get the version that supports NVCC + # Install sccache from source to get the version that supports NVCC echo "Preparing to build sccache from source" yum install -y cargo openssl-devel - echo "Checking out sccache repo" - # TODO: https://github.com/pytorch/sccache is very outdated, so let's take - # a note here to update it later with the latest code from upstream - git clone https://github.com/mozilla/sccache - cd sccache + + echo "Download sccache 0.3.0" + wget https://github.com/mozilla/sccache/archive/refs/tags/v0.3.0.tar.gz + tar xfz v0.3.0.tar.gz + + cd sccache-0.3.0 echo "Building sccache" cargo build --release cp target/release/sccache /opt/cache/bin + echo "Cleaning up" cd .. rm -rf sccache From cb2f598cf0160c4280a1bf8bb708beda71a2bfc1 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 25 Oct 2022 11:19:37 -0700 Subject: [PATCH 15/15] Merge from main --- manywheel/build_common.sh | 54 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh index c213145ef..1e2044c91 100644 --- a/manywheel/build_common.sh +++ b/manywheel/build_common.sh @@ -4,6 +4,60 @@ set -ex SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +# Courtesy of pytorch/.jenkins/pytorch/common_utils.sh +# +# - 1st arg: code to add +# - remaining args: names of traps to modify +# +trap_add() { + trap_add_cmd=$1; shift || fatal "${FUNCNAME[0]} usage error" + for trap_add_name in "$@"; do + trap -- "$( + # helper fn to get existing trap command from output + # of trap -p + extract_trap_cmd() { printf '%s\n' "$3"; } + # print existing trap command with newline + eval "extract_trap_cmd $(trap -p "${trap_add_name}")" + # print the new trap command + printf '%s\n' "${trap_add_cmd}" + )" "${trap_add_name}" \ + || fatal "unable to add to trap ${trap_add_name}" + done +} +# set the trace attribute for the above function. this is +# required to modify DEBUG or RETURN traps because functions don't +# inherit them unless the trace attribute is set +declare -f -t trap_add + +# Initialize sccache +if [[ -n "$SCCACHE_BUCKET" ]] && which sccache > /dev/null; then + # Save sccache logs to file + sccache --stop-server > /dev/null 2>&1 || true + rm -f ~/sccache_error.log || true + + export SCCACHE_IDLE_TIMEOUT=1200 + export SCCACHE_ERROR_LOG=~/sccache_error.log + export RUST_LOG=sccache::server=error + + # Report sccache stats for easier debugging + sccache --zero-stats + function sccache_epilogue() { + sccache --show-stats + sccache --stop-server || true + } + + trap_add sccache_epilogue EXIT +else + # Not using sscache if it's not setup properly + rm -f /opt/cache/bin/cc + rm -f /opt/cache/bin/c++ + rm -f /opt/cache/bin/clang + rm -f /opt/cache/bin/clang++ + rm -f /opt/cache/bin/gcc + rm -f /opt/cache/bin/g++ + + unset CMAKE_CUDA_COMPILER_LAUNCHER +fi # Require only one python installation if [[ -z "$DESIRED_PYTHON" ]]; then