From c08f3824cd82ca2dfc51a0840d0cdea4816080f3 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sat, 25 Feb 2017 00:00:47 -0800 Subject: [PATCH] travis: Make more network requests retryable This commit attempts to move more network operations to being retryable through various operations. For example git submodule updates, downloading snapshots, etc, are now all in retryable steps. Hopefully this commit can cut down on the number of network failures we've been seeing! --- .travis.yml | 4 ++-- appveyor.yml | 8 ++++---- src/bootstrap/mk/Makefile.in | 2 ++ src/ci/docker/run.sh | 6 ++++-- src/ci/run.sh | 10 +++++++--- src/ci/shared.sh | 29 +++++++++++++++++++++++++++++ 6 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 src/ci/shared.sh diff --git a/.travis.yml b/.travis.yml index cc032edb90873..9e90a1d2ddbf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,7 @@ matrix: os: osx osx_image: xcode8.2 install: &osx_install_sccache > - curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin && + travis_retry curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin && chmod +x /usr/local/bin/sccache - env: > RUST_CHECK_TARGET=check @@ -63,7 +63,7 @@ matrix: os: osx osx_image: xcode8.2 install: > - curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin && + travis_retry curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin && chmod +x /usr/local/bin/sccache && brew uninstall --ignore-dependencies openssl && brew install openssl --universal --without-test diff --git a/appveyor.yml b/appveyor.yml index ad6b53162269e..f76fc0f6466ff 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -103,7 +103,7 @@ install: # /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we # move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe` # file exists in there (which it doesn't by default). - - if defined MINGW_URL appveyor DownloadFile %MINGW_URL%/%MINGW_ARCHIVE% + - if defined MINGW_URL appveyor-retry appveyor DownloadFile %MINGW_URL%/%MINGW_ARCHIVE% - if defined MINGW_URL 7z x -y %MINGW_ARCHIVE% > nul - if defined MINGW_URL set PATH=%CD%\%MINGW_DIR%\bin;C:\msys64\usr\bin;%PATH% @@ -115,12 +115,12 @@ install: - set PATH=C:\Python27;%PATH% # Download and install sccache - - appveyor DownloadFile https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-pc-windows-msvc + - appveyor-retry appveyor DownloadFile https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-pc-windows-msvc - mv 2017-02-24-sccache-x86_64-pc-windows-msvc sccache - set PATH=%PATH%;%CD% # Install InnoSetup to get `iscc` used to produce installers - - choco install -y InnoSetup + - appveyor-retry choco install -y InnoSetup - set PATH="C:\Program Files (x86)\Inno Setup 5";%PATH% # Help debug some handle issues on AppVeyor @@ -131,7 +131,7 @@ install: - handle.exe -accepteula -help test_script: - - git submodule update --init + - appveyor-retry sh -c 'git submodule deinit -f . && git submodule update --init' - set SRC=. - set NO_CCACHE=1 - sh src/ci/run.sh diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 536095503e0da..457ac825832ce 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -71,6 +71,8 @@ install: $(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS) tidy: $(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS) +prepare: + $(Q)$(BOOTSTRAP) build nonexistent/path/to/trigger/cargo/metadata check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu: $(Q)$(BOOTSTRAP) test --target arm-linux-androideabi diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 892c5baa5c64b..1e61f2169106c 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2016 The Rust Project Developers. See the COPYRIGHT # file at the top-level directory of this distribution and at # http://rust-lang.org/COPYRIGHT. @@ -19,7 +19,9 @@ ci_dir="`dirname $docker_dir`" src_dir="`dirname $ci_dir`" root_dir="`dirname $src_dir`" -docker \ +source "$ci_dir/shared.sh" + +retry docker \ build \ --rm \ -t rust-ci \ diff --git a/src/ci/run.sh b/src/ci/run.sh index c4b2556599366..53d16f6239e66 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2016 The Rust Project Developers. See the COPYRIGHT # file at the top-level directory of this distribution and at # http://rust-lang.org/COPYRIGHT. @@ -20,6 +20,9 @@ if [ "$NO_CHANGE_USER" = "" ]; then fi fi +ci_dir=`cd $(dirname $0) && pwd` +source "$ci_dir/shared.sh" + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-quiet-tests" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules" @@ -54,9 +57,8 @@ else fi fi -set -ex - $SRC/configure $RUST_CONFIGURE_ARGS +retry make prepare if [ "$TRAVIS_OS_NAME" = "osx" ]; then ncpus=$(sysctl -n hw.ncpu) @@ -64,6 +66,8 @@ else ncpus=$(grep processor /proc/cpuinfo | wc -l) fi +set -x + if [ ! -z "$SCRIPT" ]; then sh -x -c "$SCRIPT" else diff --git a/src/ci/shared.sh b/src/ci/shared.sh new file mode 100644 index 0000000000000..ecd9b7e98a482 --- /dev/null +++ b/src/ci/shared.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright 2016 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +# See http://unix.stackexchange.com/questions/82598 +function retry { + local n=1 + local max=5 + local delay=15 + while true; do + echo "Attempting:" "$@" + "$@" && break || { + if [[ $n -lt $max ]]; then + ((n++)) + echo "Command failed. Attempt $n/$max:" + else + echo "The command has failed after $n attempts." + exit 1 + fi + } + done +}