From 744b1f3cd3088ab417edec466dbf191c74955214 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 14 Oct 2021 15:54:07 -0400 Subject: [PATCH] CI (Buildkite): Add a once-daily scheduled job that builds Julia with `USE_BINARYBUILDER=0` (and runs the test suite) on linux64 (#42592) (cherry picked from commit 4c222c9071a142c9c6b3685eecc0bf88ee0ef85d) --- .../main/launch_unsigned_builders.yml | 4 +- .../main/platforms/package_linux.arches | 14 +++--- .../main/platforms/package_linux.yml | 16 +++---- .../pipelines/main/platforms/platforms.sh | 26 ----------- .../main/platforms/tester_linux.arches | 18 ++++---- .../pipelines/main/platforms/tester_linux.yml | 20 ++++----- .buildkite/pipelines/scheduled/0_webui.yml | 4 +- .../scheduled/launch_unsigned_jobs.yml | 8 ++++ .../no_bb/no_bb_package_linux.arches | 2 + .../scheduled/no_bb/no_bb_tester_linux.arches | 2 + .buildkite/utilities/platforms/platforms.sh | 43 +++++++++++++++++++ 11 files changed, 93 insertions(+), 64 deletions(-) delete mode 100755 .buildkite/pipelines/main/platforms/platforms.sh create mode 100644 .buildkite/pipelines/scheduled/launch_unsigned_jobs.yml create mode 100644 .buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches create mode 100644 .buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches create mode 100755 .buildkite/utilities/platforms/platforms.sh diff --git a/.buildkite/pipelines/main/launch_unsigned_builders.yml b/.buildkite/pipelines/main/launch_unsigned_builders.yml index 4ff6e310e3c126..2b6794ed13bd14 100644 --- a/.buildkite/pipelines/main/launch_unsigned_builders.yml +++ b/.buildkite/pipelines/main/launch_unsigned_builders.yml @@ -22,8 +22,8 @@ steps: buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml # Launch all of the platform jobs. - bash .buildkite/pipelines/main/platforms/platforms.sh package_linux - bash .buildkite/pipelines/main/platforms/platforms.sh tester_linux + bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml + bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml # Launch the `whitespace` job last. Uploading it last actually causes it to start # first. We want this job to start first because we want it to finish as quickly diff --git a/.buildkite/pipelines/main/platforms/package_linux.arches b/.buildkite/pipelines/main/platforms/package_linux.arches index 58d5b87f4c3a39..89e3c9fd7dc4d9 100644 --- a/.buildkite/pipelines/main/platforms/package_linux.arches +++ b/.buildkite/pipelines/main/platforms/package_linux.arches @@ -1,7 +1,7 @@ -# PLATFORM ARCH ARCH_LABEL ROOTFS_ARCH ALLOW_FAIL TIMEOUT ROOTFS_TAG ROOTFS_TREE -# linux aarch64 aarch64 aarch64 false 60 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff -# linux armv7l armv7l armv7l false 60 v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25 -linux 32 32 i686 false 60 v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32 -# linux ppc64le ppc64le powerpc64le false 60 v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7 -linux 64 64 x86_64 false 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 -musl 64 64 x86_64 false 60 v3.19 e6a2730e37c386c46915b2650d6aaaa398195152 +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH +# linux _aarch64 false _aarch64 aarch64 none 60 no no no v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff +# linux _armv7l false _armv7l armv7l none 60 no no no v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25 +linux 32 false 32 i686 none 60 no no no v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32 +# linux _ppc64le false _ppc64le powerpc64le none 60 no no no v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7 +linux 64 false 64 x86_64 none 60 no no no v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 +musl 64 false 64 x86_64 none 60 no no no v3.19 e6a2730e37c386c46915b2650d6aaaa398195152 diff --git a/.buildkite/pipelines/main/platforms/package_linux.yml b/.buildkite/pipelines/main/platforms/package_linux.yml index 1a6f15b8fe8123..66c0f4784d942e 100644 --- a/.buildkite/pipelines/main/platforms/package_linux.yml +++ b/.buildkite/pipelines/main/platforms/package_linux.yml @@ -4,16 +4,16 @@ agents: sandbox.jl: "true" os: "linux" steps: - - label: "package_${PLATFORM?}${ARCH_LABEL?}" - key: package_${PLATFORM?}${ARCH_LABEL?} + - label: "package_${PLATFORM?}${LABEL?}" + key: package_${PLATFORM?}${LABEL?} plugins: - JuliaCI/julia#v1: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.6' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz - rootfs_treehash: "${ROOTFS_TREE?}" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz + rootfs_treehash: "${ROOTFS_HASH?}" uid: 1000 gid: 1000 workspaces: @@ -26,7 +26,7 @@ steps: SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}` ARTIFACT_FILE_EXTENSION="tar.gz" ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}" - JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-` + JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME ${MAKE_FLAGS?} | cut -c27- | tr -s ' '` JULIA_BINARYDIST="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}" echo "The full commit is: $${BUILDKITE_COMMIT:?}" @@ -35,16 +35,16 @@ steps: echo "--- Build Julia from source" rm -rf $${ARTIFACT_FILENAME:?} - make --output-sync -j 8 + make --output-sync -j 8 ${MAKE_FLAGS?} - echo "--- Make sure that the working directory is clean" + echo "--- Check that the working directory is clean" if [ -z "$(git status --short)" ]; then echo "INFO: The working directory is clean."; else echo "ERROR: The working directory is dirty."; echo "Output of git status:"; git status; exit 1; fi echo "--- Print Julia version info" ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' echo "--- Create build artifacts" - make --output-sync -j 8 binary-dist + make --output-sync -j 8 binary-dist ${MAKE_FLAGS?} ls -l $${JULIA_BINARYDIST:?} if [[ "$${JULIA_BINARYDIST:?}" != "$${ARTIFACT_FILENAME:?}" ]]; then mv $${JULIA_BINARYDIST:?} $${ARTIFACT_FILENAME:?} diff --git a/.buildkite/pipelines/main/platforms/platforms.sh b/.buildkite/pipelines/main/platforms/platforms.sh deleted file mode 100755 index 760d65c4e842f7..00000000000000 --- a/.buildkite/pipelines/main/platforms/platforms.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -OS="$1" - -cat "$SCRIPT_DIR/$OS.arches" | tr -s ' ' | while read _line; do - # Remove whitespace from the beginning and end of each line - line=`echo $_line | tr -s ' '` - - # Skip all lines that begin with `#` - if [[ $line == \#* ]]; then - continue - fi - - export PLATFORM=`echo $line | cut -d ' ' -f 1` - export ARCH=`echo $line | cut -d ' ' -f 2` - export ARCH_LABEL=`echo $line | cut -d ' ' -f 3` - export ROOTFS_ARCH=`echo $line | cut -d ' ' -f 4` - export ALLOW_FAIL=`echo $line | cut -d ' ' -f 5` - export TIMEOUT=`echo $line | cut -d ' ' -f 6` - export ROOTFS_TAG=`echo $line | cut -d ' ' -f 7` - export ROOTFS_TREE=`echo $line | cut -d ' ' -f 8` - echo "Launching: $OS $PLATFORM $ARCH $ARCH_LABEL $ROOTFS_ARCH $ALLOW_FAIL $TIMEOUT" - buildkite-agent pipeline upload "$SCRIPT_DIR/$OS.yml" -done diff --git a/.buildkite/pipelines/main/platforms/tester_linux.arches b/.buildkite/pipelines/main/platforms/tester_linux.arches index 33216f39a5eb8a..9d07ff26d27751 100644 --- a/.buildkite/pipelines/main/platforms/tester_linux.arches +++ b/.buildkite/pipelines/main/platforms/tester_linux.arches @@ -1,9 +1,9 @@ -# PLATFORM ARCH ARCH_LABEL ROOTFS_ARCH ALLOW_FAIL TIMEOUT ROOTFS_TAG ROOTFS_TREE -# linux aarch64 aarch64 aarch64 false 60 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff -# linux armv7l armv7l armv7l false 60 v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25 -linux 32 32 i686 false 60 v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32 -# linux ppc64le ppc64le powerpc64le false 60 v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7 -linux 64 64_rr x86_64 false 180 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 -linux 64 64_st x86_64 false 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 -linux 64 64_mt x86_64 false 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 -musl 64 64 x86_64 true 60 v3.19 e6a2730e37c386c46915b2650d6aaaa398195152 +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH +# linux _aarch64 false _aarch64 aarch64 none 60 no no no v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff +# linux _armv7l false _armv7l armv7l none 60 no no no v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25 +linux 32 false 32 i686 none 60 no no no v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32 +# linux _ppc64le false _ppc64le powerpc64le none 60 no no no v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7 +linux 64_rr false 64 x86_64 none 60 yes no no v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 +linux 64_st false 64 x86_64 none 60 no yes no v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 +linux 64_mt false 64 x86_64 none 60 no no yes v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 +musl 64 true 64 x86_64 none 60 no no no v3.19 e6a2730e37c386c46915b2650d6aaaa398195152 diff --git a/.buildkite/pipelines/main/platforms/tester_linux.yml b/.buildkite/pipelines/main/platforms/tester_linux.yml index a70dd5911ad9b9..8016d85523a8b2 100644 --- a/.buildkite/pipelines/main/platforms/tester_linux.yml +++ b/.buildkite/pipelines/main/platforms/tester_linux.yml @@ -4,8 +4,8 @@ agents: sandbox.jl: "true" os: "linux" steps: - - label: "tester_${PLATFORM?}${ARCH_LABEL?}" - key: tester_${PLATFORM?}${ARCH_LABEL?} + - label: "tester_${PLATFORM?}${LABEL?}" + key: tester_${PLATFORM?}${LABEL?} depends_on: package_${PLATFORM?}${ARCH?} plugins: - JuliaCI/julia#v1: @@ -13,9 +13,9 @@ steps: persist_depot_dirs: packages,artifacts,compiled version: '1.6' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz - # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz - rootfs_treehash: "${ROOTFS_TREE?}" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz + # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz + rootfs_treehash: "${ROOTFS_HASH?}" uid: 1000 gid: 1000 workspaces: @@ -57,7 +57,7 @@ steps: export OPENBLAS_NUM_THREADS=8 export TESTS="[\"all\"]" - if [[ "$${BUILDKITE_STEP_KEY:?}" == "tester_linux64_rr" ]]; then + if [[ "${IS_RR?}" == "yes" ]]; then export JULIA_BINARY_UNDER_RR="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}" export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY_UNDER_RR:?}" @@ -69,11 +69,11 @@ steps: export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY:?}" export NCORES_FOR_TESTS="Sys.CPU_THREADS" - if [[ "$${BUILDKITE_STEP_KEY:?}" == "tester_linux64_st" ]]; then - # "_st" = single-threaded + if [[ "${IS_ST?}" == "yes" ]]; then + # "ST" = single-threaded export JULIA_NUM_THREADS=1 - elif [[ "$${BUILDKITE_STEP_KEY:?}" == "tester_linux64_mt" ]]; then - # "_mt" = multi-threaded + elif [[ "${IS_MT?}" == "yes" ]]; then + # "MT" = multi-threaded export JULIA_NUM_THREADS=16 # We do not run the Distributed test suite when multithreading is enabled. diff --git a/.buildkite/pipelines/scheduled/0_webui.yml b/.buildkite/pipelines/scheduled/0_webui.yml index 03024bd51c6801..78031b49d9f17d 100644 --- a/.buildkite/pipelines/scheduled/0_webui.yml +++ b/.buildkite/pipelines/scheduled/0_webui.yml @@ -13,8 +13,8 @@ steps: # Build secrets will not be available in these pipelines (or their children) # but some of our signed pipelines can wait upon the completion of these unsigned # pipelines. - # unsigned_pipelines: - # - .buildkite/pipelines/scheduled/launch_unsigned_builders.yml + unsigned_pipelines: + - .buildkite/pipelines/scheduled/launch_unsigned_jobs.yml # Our signed pipelines must have a `signature` or `signature_file` parameter that # verifies the treehash of the pipeline itself and the inputs listed in `inputs` diff --git a/.buildkite/pipelines/scheduled/launch_unsigned_jobs.yml b/.buildkite/pipelines/scheduled/launch_unsigned_jobs.yml new file mode 100644 index 00000000000000..300c8d8466aea1 --- /dev/null +++ b/.buildkite/pipelines/scheduled/launch_unsigned_jobs.yml @@ -0,0 +1,8 @@ +steps: + - label: ":buildkite: Launch unsigned jobs" + commands: | + # Launch all of the `USE_BINARYBUILDER=0` jobs. + bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml + bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml + agents: + queue: julia diff --git a/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches b/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches new file mode 100644 index 00000000000000..ce76950e626f7d --- /dev/null +++ b/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches @@ -0,0 +1,2 @@ +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH +linux 64_no_bb false 64_no_bb x86_64 USE_BINARYBUILDER=0 240 no no no v4.4 ec3873fc1756ecb4c815e8785e9ca45cf54c68b6 diff --git a/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches b/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches new file mode 100644 index 00000000000000..9cbe03b0c0d437 --- /dev/null +++ b/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches @@ -0,0 +1,2 @@ +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH +linux 64_st_no_bb false 64_no_bb x86_64 none 60 no yes no v4.4 ec3873fc1756ecb4c815e8785e9ca45cf54c68b6 diff --git a/.buildkite/utilities/platforms/platforms.sh b/.buildkite/utilities/platforms/platforms.sh new file mode 100755 index 00000000000000..87daa8a2a9e285 --- /dev/null +++ b/.buildkite/utilities/platforms/platforms.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +ARCHES="$1" +YAML="$2" + +if [[ ! -f "${ARCHES:?}" ]] ; then + echo "File does not exist: ${ARCHES:?}" + exit 1 +fi + +if [[ ! -f "${YAML:?}" ]] ; then + echo "File does not exist: ${YAML:?}" + exit 1 +fi + +cat "${ARCHES:?}" | tr -s ' ' | while read _line; do + # Remove whitespace from the beginning and end of each line + line=`echo $_line | tr -s ' '` + + # Skip all lines that begin with `#` + if [[ $line == \#* ]]; then + continue + fi + + export PLATFORM=`echo $line | cut -d ' ' -f 1 | tr -s ' '` + export LABEL=`echo $line | cut -d ' ' -f 2 | tr -s ' '` + export ALLOW_FAIL=`echo $line | cut -d ' ' -f 3 | tr -s ' '` + export ARCH=`echo $line | cut -d ' ' -f 4 | tr -s ' '` + export ARCH_ROOTFS=`echo $line | cut -d ' ' -f 5 | tr -s ' '` + export MAKE_FLAGS=`echo $line | cut -d ' ' -f 6 | tr -s ' '` + export TIMEOUT=`echo $line | cut -d ' ' -f 7 | tr -s ' '` + export IS_RR=`echo $line | cut -d ' ' -f 8 | tr -s ' '` + export IS_ST=`echo $line | cut -d ' ' -f 9 | tr -s ' '` + export IS_MT=`echo $line | cut -d ' ' -f 10 | tr -s ' '` + export ROOTFS_TAG=`echo $line | cut -d ' ' -f 11 | tr -s ' '` + export ROOTFS_HASH=`echo $line | cut -d ' ' -f 12 | tr -s ' '` + + if [[ "${MAKE_FLAGS:?}" == "none" ]]; then + export MAKE_FLAGS="" + fi + + buildkite-agent pipeline upload "${YAML:?}" +done