CI (Buildkite): add a separate Buildkite job that runs the tests of t…

…he network-related stdlibs (e.g. Downloads.jl), and automatically retries that job up to a maximum number of tries (#42861) (cherry picked from commit 3003742)
JuliaLang · Nov 12, 2021 · 655163f · 655163f
1 parent 09a49c6
commit 655163f
Show file tree

Hide file tree

Showing 8 changed files with 112 additions and 69 deletions.
diff --git a/.buildkite/pipelines/main/platforms/package_linux.arches b/.buildkite/pipelines/main/platforms/package_linux.arches
@@ -1,7 +1,7 @@
-# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
-# linux _aarch64 false  _aarch64 aarch64 none  60 no  no  no v0.0 0000000000000000000000000000000000000000
-# linux _armv7l  false  _armv7l armv7l none 60 no no no v0.0 0000000000000000000000000000000000000000
-linux  32  false 32  i686 none  60 no  no no  v4.8  b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
-# linux _ppc64le false  _ppc64le powerpc64le  none  60  no no  no v0.0 0000000000000000000000000000000000000000
-linux  64  false 64 x86_64  none 60 no no no v4.8  2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-musl 64 false 64 x86_64 none 60 no  no no v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
+# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
+linux  32  .  32 i686   .  .  .  . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
+linux  64 .  64 x86_64 . . .  .  . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+# linux _aarch64 . _aarch64 aarch64 .  . .   .  .  . . . .... ........................................
+# linux _armv7l  .  _armv7l  armv7l . .  .   .  . . . . .... ........................................
+# linux _ppc64le . _ppc64le powerpc64le .  . . . .  . . . .... ........................................
+musl 64 .  64 x86_64 . .  .  . . . . .  v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
diff --git a/.buildkite/pipelines/main/platforms/package_linux.yml b/.buildkite/pipelines/main/platforms/package_linux.yml
@@ -19,7 +19,7 @@ steps:
  workspaces:
  # Include `/cache/repos` so that our `git` version introspection works.
  - "/cache/repos:/cache/repos"
- timeout_in_minutes: ${TIMEOUT?}
+ timeout_in_minutes: ${TIMEOUT_BK?}
  commands: |
  echo "--- Print the full and short commit hashes"
  SHORT_COMMIT_LENGTH=10

diff --git a/.buildkite/pipelines/main/platforms/tester_linux.arches b/.buildkite/pipelines/main/platforms/tester_linux.arches
@@ -1,9 +1,14 @@
-# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
-# linux _aarch64 false _aarch64 aarch64 none 60 no no no v0.0 0000000000000000000000000000000000000000
-# linux _armv7l false _armv7l armv7l none 60 no no no v0.0 0000000000000000000000000000000000000000
-linux 32 false 32 i686 none 60 no no no v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
-# linux _ppc64le false _ppc64le powerpc64le none 60 no no no v0.0 0000000000000000000000000000000000000000
-linux 64_rr false 64 x86_64 none 180 yes no no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-linux 64_st false 64 x86_64 none 60 no yes no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-linux 64_mt false 64 x86_64 none 60 no no yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-musl 64 true 64 x86_64 none 60 no no no v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
+# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
+
+linux 32 . 32 i686 no-net . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
+linux 32_net . 32 i686 net . . . 3 . yes . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
+
+linux 64_rr . 64 x86_64 no-net . 180 120 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux 64_net . 64 x86_64 net . 120 60 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux 64_st . 64 x86_64 no-net . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux 64_mt . 64 x86_64 no-net . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+
+# linux _aarch64 . _aarch64 aarch64 . . . . . . . . .... ........................................
+# linux _armv7l . _armv7l armv7l . . . . . . . . .... ........................................
+# linux _ppc64le . _ppc64le powerpc64le . . . . . . . . .... ........................................
+musl 64 true 64 x86_64 . . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
diff --git a/.buildkite/pipelines/main/platforms/tester_linux.yml b/.buildkite/pipelines/main/platforms/tester_linux.yml
@@ -23,7 +23,11 @@ steps:
  - "/cache/repos:/cache/repos"
  env:
  JULIA_SHELL: "/bin/bash"
- timeout_in_minutes: ${TIMEOUT?}
+ timeout_in_minutes: ${TIMEOUT_BK?}
+ retry:
+ automatic:
+ - exit_status: "*"
+ limit: ${RETRIES?}
  soft_fail: ${ALLOW_FAIL?}
  commands: |
  echo "--- Print the full and short commit hashes"
@@ -55,45 +59,42 @@ steps:
  echo "--- Set some environment variables in preparation for running the Julia test suite"
  unset JULIA_DEPOT_PATH
  export OPENBLAS_NUM_THREADS=8
- export TESTS="[\"all\"]"
 
- if [[ "${IS_RR?}" == "yes" ]]; then
- export JULIA_BINARY_UNDER_RR="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
+ if [[ "${GROUP?}" == "all" ]]; then
+ export TESTS="all"
+ elif [[ "${GROUP?}" == "net" ]]; then
+ export TESTS="Downloads LibGit2 LibGit2/online download"
+ elif [[ "${GROUP?}" == "no-net" ]]; then
+ export TESTS="all --skip Downloads LibGit2/online download"
+ else
+ echo "Invalid value for GROUP: ${GROUP?}"
+ exit 1
+ fi
 
- export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY_UNDER_RR:?}"
- export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
+ export JULIA_TEST_RR_TIMEOUT="${TIMEOUT_RR?}"
 
- # For the `rr` job, we disable multi-threading.
- export JULIA_NUM_THREADS=1
+ if [[ "${IS_RR?}" == "yes" ]]; then
+ export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
+ export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
  else
- export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY:?}"
+ export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?}"
  export NCORES_FOR_TESTS="Sys.CPU_THREADS"
+ fi
 
- if [[ "${IS_ST?}" == "yes" ]]; then
- # "ST" = single-threaded
- export JULIA_NUM_THREADS=1
- elif [[ "${IS_MT?}" == "yes" ]]; then
- # "MT" = multi-threaded
- export JULIA_NUM_THREADS=16
+ if [[ "${IS_ST?}" == "yes" ]]; then
+ export JULIA_NUM_THREADS=1
+ fi
 
- if [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-release-1-dot-6" ]]; then
- # On Julia 1.6, we skip the Distributed test suite if multithreading is enabled.
- export TESTS="[\"all\", \"--skip\", \"Distributed\"]"
- elif [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-release-1-dot-7" ]]; then
- # On Julia 1.7, we skip the Distributed test suite if multithreading is enabled.
- export TESTS="[\"all\", \"--skip\", \"Distributed\"]"
- else
- # On Julia 1.8 and later, we do not skip any test suites.
- export TESTS="[\"all\"]"
- fi
- fi
+ if [[ "${IS_MT?}" == "yes" ]]; then
+ export JULIA_NUM_THREADS=16
  fi
 
- echo "JULIA_BINARY_FOR_TESTS is: $${JULIA_BINARY_FOR_TESTS:?}"
- echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # this variable might not be set
+ echo "JULIA_CMD_FOR_TESTS is:  $${JULIA_CMD_FOR_TESTS:?}"
+ echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set
  echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}"
  echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}"
+ echo "GROUP is: ${GROUP?}"
  echo "TESTS is: $${TESTS:?}"
 
  echo "--- Run the Julia test suite"
- $${JULIA_BINARY_FOR_TESTS:?} -e "Base.runtests($${TESTS:?}; ncores = $${NCORES_FOR_TESTS:?})"
+ $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})"
diff --git a/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches b/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches
@@ -1,2 +1,2 @@
-# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
-linux 64_no_bb false 64_no_bb x86_64 USE_BINARYBUILDER=0 240 no no no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
+linux 64_no_bb false 64_no_bb x86_64 . USE_BINARYBUILDER=0 180 .  . . . .  v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
diff --git a/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches b/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches
@@ -1,2 +1,3 @@
-# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
-linux 64_st_no_bb false 64_no_bb x86_64 none 60 no yes no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
+linux 64_no_bb . 64_no_bb x86_64 no-net . 180 120 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux 64_no_bb_net . 64_no_bb x86_64 net . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
diff --git a/.buildkite/utilities/platforms/platforms.sh b/.buildkite/utilities/platforms/platforms.sh
@@ -4,40 +4,75 @@ ARCHES="$1"
 YAML="$2"
 
 if [[ ! -f "${ARCHES:?}" ]] ; then
- echo "File does not exist: ${ARCHES:?}"
+ echo "Arches file does not exist: ${ARCHES:?}"
  exit 1
 fi
 
 if [[ ! -f "${YAML:?}" ]] ; then
- echo "File does not exist: ${YAML:?}"
+ echo "YAML file does not exist: ${YAML:?}"
  exit 1
 fi
 
 cat "${ARCHES:?}" | tr -s ' ' | while read _line; do
  # Remove whitespace from the beginning and end of each line
  line=`echo $_line | tr -s ' '`
 
- # Skip all lines that begin with `#`
+ # Skip any line that begins with the `#` character
  if [[ $line == \#* ]]; then
  continue
  fi
 
+ # Skip any empty line
+ if [[ $line == "" ]]; then
+ continue
+ fi
+
  export PLATFORM=`echo $line | cut -d ' ' -f 1 | tr -s ' '`
  export LABEL=`echo $line | cut -d ' ' -f 2 | tr -s ' '`
  export ALLOW_FAIL=`echo $line | cut -d ' ' -f 3 | tr -s ' '`
  export ARCH=`echo $line | cut -d ' ' -f 4 | tr -s ' '`
  export ARCH_ROOTFS=`echo $line | cut -d ' ' -f 5 | tr -s ' '`
- export MAKE_FLAGS=`echo $line | cut -d ' ' -f 6 | tr -s ' '`
- export TIMEOUT=`echo $line | cut -d ' ' -f 7 | tr -s ' '`
- export IS_RR=`echo $line | cut -d ' ' -f 8 | tr -s ' '`
- export IS_ST=`echo $line | cut -d ' ' -f 9 | tr -s ' '`
- export IS_MT=`echo $line | cut -d ' ' -f 10 | tr -s ' '`
- export ROOTFS_TAG=`echo $line | cut -d ' ' -f 11 | tr -s ' '`
- export ROOTFS_HASH=`echo $line | cut -d ' ' -f 12 | tr -s ' '`
-
- if [[ "${MAKE_FLAGS:?}" == "none" ]]; then
+ export GROUP=`echo $line | cut -d ' ' -f 6 | tr -s ' '`
+ export MAKE_FLAGS=`echo $line | cut -d ' ' -f 7 | tr -s ' '`
+ export TIMEOUT_BK=`echo $line | cut -d ' ' -f 8 | tr -s ' '`
+ export TIMEOUT_RR=`echo $line | cut -d ' ' -f 9 | tr -s ' '`
+ export RETRIES=`echo $line | cut -d ' ' -f 10 | tr -s ' '`
+ export IS_RR=`echo $line | cut -d ' ' -f 11 | tr -s ' '`
+ export IS_ST=`echo $line | cut -d ' ' -f 12 | tr -s ' '`
+ export IS_MT=`echo $line | cut -d ' ' -f 13 | tr -s ' '`
+ export ROOTFS_TAG=`echo $line | cut -d ' ' -f 14 | tr -s ' '`
+ export ROOTFS_HASH=`echo $line | cut -d ' ' -f 15 | tr -s ' '`
+
+ if [[ "${ALLOW_FAIL:?}" == "." ]]; then
+ export ALLOW_FAIL="false"
+ fi
+
+ if [[ "${GROUP:?}" == "." ]]; then
+ export GROUP="all"
+ fi
+
+ if [[ "${MAKE_FLAGS:?}" == "." ]]; then
  export MAKE_FLAGS=""
  fi
 
+ if [[ "${TIMEOUT_BK:?}" == "." ]]; then
+ export TIMEOUT_BK="60"
+ fi
+
+ if [[ "${TIMEOUT_RR:?}" == "." ]]; then
+ export TIMEOUT_RR="30"
+ fi
+
+ if [[ "${RETRIES:?}" == "." ]]; then
+ export RETRIES="0"
+ fi
+
+ if [[ "${IS_ST:?}" == "yes" ]]; then
+ if [[ "${IS_MT:?}" == "yes" ]]; then
+ echo "You cannot set both IS_ST and IS_MT to yes"
+ exit 1
+ fi
+ fi
+
  buildkite-agent pipeline upload "${YAML:?}"
 done
diff --git a/.buildkite/utilities/rr/rr_capture.jl b/.buildkite/utilities/rr/rr_capture.jl
@@ -55,13 +55,14 @@ end
 
 @info "We will run the command under rr"
 
-const build_number = get_from_env("BUILDKITE_BUILD_NUMBER")
-const job_name = get_from_env("BUILDKITE_STEP_KEY")
-const commit_full = get_from_env("BUILDKITE_COMMIT")
-const commit_short = first(commit_full, 10)
-const timeout_minutes = 120
-const JULIA_TEST_NUM_CORES = get(ENV, "JULIA_TEST_NUM_CORES", "8")
-const julia_test_num_cores_int = parse(Int, JULIA_TEST_NUM_CORES)
+const build_number = get_from_env("BUILDKITE_BUILD_NUMBER")
+const job_name = get_from_env("BUILDKITE_STEP_KEY")
+const commit_full = get_from_env("BUILDKITE_COMMIT")
+const commit_short = first(commit_full, 10)
+const JULIA_TEST_RR_TIMEOUT = get(ENV, "JULIA_TEST_RR_TIMEOUT", "120")
+const timeout_minutes = parse(Int, JULIA_TEST_RR_TIMEOUT)
+const JULIA_TEST_NUM_CORES = get(ENV, "JULIA_TEST_NUM_CORES", "8")
+const julia_test_num_cores_int = parse(Int, JULIA_TEST_NUM_CORES)
 const num_cores = min(
  8,
  Sys.CPU_THREADS,