CI (Buildkite): Add a once-daily scheduled job that builds Julia with…

… `USE_BINARYBUILDER=0` (and runs the test suite) on linux64 (#42592)

(cherry picked from commit 4c222c9)

.buildkite/pipelines/main/launch_unsigned_builders.yml

@@ -22,8 +22,8 @@ steps:
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml
 
       # Launch all of the platform jobs.
-      bash .buildkite/pipelines/main/platforms/platforms.sh package_linux
-      bash .buildkite/pipelines/main/platforms/platforms.sh tester_linux
+      bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml
+      bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml
 
       # Launch the `whitespace` job last. Uploading it last actually causes it to start
       # first. We want this job to start first because we want it to finish as quickly

.buildkite/pipelines/main/platforms/package_linux.arches

@@ -1,7 +1,7 @@
-# PLATFORM      ARCH        ARCH_LABEL      ROOTFS_ARCH      ALLOW_FAIL      TIMEOUT      ROOTFS_TAG      ROOTFS_TREE
-# linux         aarch64     aarch64         aarch64          false           60           v3.2            0566841e29f0f9880541c26a6595fd5ce0beb5ff
-# linux         armv7l      armv7l          armv7l           false           60           v3.2            fb359370b052a47ce5c84cc6b4a7a03ed7053b25
-linux           32          32              i686             false           60           v3.2            209c4db679a515befd7fb50ecc6bfbecf7ec3d32
-# linux         ppc64le     ppc64le         powerpc64le      false           60           v3.2            c03a0158b19d48ac84b426834fce0d3584cdd0c7
-linux           64          64              x86_64           false           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
-musl            64          64              x86_64           false           60           v3.19           e6a2730e37c386c46915b2650d6aaaa398195152
+# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
+# linux       _aarch64    false         _aarch64    aarch64        none           60         no       no       no       v3.2          0566841e29f0f9880541c26a6595fd5ce0beb5ff
+# linux       _armv7l     false         _armv7l     armv7l         none           60         no       no       no       v3.2          fb359370b052a47ce5c84cc6b4a7a03ed7053b25
+linux         32          false         32          i686           none           60         no       no       no       v3.2          209c4db679a515befd7fb50ecc6bfbecf7ec3d32
+# linux       _ppc64le    false         _ppc64le    powerpc64le    none           60         no       no       no       v3.2          c03a0158b19d48ac84b426834fce0d3584cdd0c7
+linux         64          false         64          x86_64         none           60         no       no       no       v3.2          474bf61a926b2d7fcf202284d59d4b11a04601d7
+musl          64          false         64          x86_64         none           60         no       no       no       v3.19         e6a2730e37c386c46915b2650d6aaaa398195152

.buildkite/pipelines/main/platforms/package_linux.yml

@@ -4,16 +4,16 @@ agents:
   sandbox.jl: "true"
   os: "linux"
 steps:
-  - label: "package_${PLATFORM?}${ARCH_LABEL?}"
-    key: package_${PLATFORM?}${ARCH_LABEL?}
+  - label: "package_${PLATFORM?}${LABEL?}"
+    key: package_${PLATFORM?}${LABEL?}
     plugins:
       - JuliaCI/julia#v1:
           # Drop default "registries" directory, so it is not persisted from execution to execution
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz
-          rootfs_treehash: "${ROOTFS_TREE?}"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz
+          rootfs_treehash: "${ROOTFS_HASH?}"
           uid: 1000
           gid: 1000
           workspaces:
@@ -26,7 +26,7 @@ steps:
       SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}`
       ARTIFACT_FILE_EXTENSION="tar.gz"
       ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}"
-      JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-`
+      JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME ${MAKE_FLAGS?} | cut -c27- | tr -s ' '`
       JULIA_BINARYDIST="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}"
 
       echo "The full commit is:                     $${BUILDKITE_COMMIT:?}"
@@ -35,16 +35,16 @@ steps:
 
       echo "--- Build Julia from source"
       rm -rf $${ARTIFACT_FILENAME:?}
-      make --output-sync -j 8
+      make --output-sync -j 8 ${MAKE_FLAGS?}
 
-      echo "--- Make sure that the working directory is clean"
+      echo "--- Check that the working directory is clean"
       if [ -z "$(git status --short)" ]; then echo "INFO: The working directory is clean."; else echo "ERROR: The working directory is dirty."; echo "Output of git status:"; git status; exit 1; fi
 
       echo "--- Print Julia version info"
       ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
 
       echo "--- Create build artifacts"
-      make --output-sync -j 8 binary-dist
+      make --output-sync -j 8 binary-dist ${MAKE_FLAGS?}
       ls -l $${JULIA_BINARYDIST:?}
       if [[ "$${JULIA_BINARYDIST:?}" != "$${ARTIFACT_FILENAME:?}" ]]; then
           mv $${JULIA_BINARYDIST:?} $${ARTIFACT_FILENAME:?}

.buildkite/pipelines/main/platforms/tester_linux.arches

@@ -1,9 +1,9 @@
-# PLATFORM      ARCH        ARCH_LABEL      ROOTFS_ARCH      ALLOW_FAIL      TIMEOUT      ROOTFS_TAG      ROOTFS_TREE
-# linux         aarch64     aarch64         aarch64          false           60           v3.2            0566841e29f0f9880541c26a6595fd5ce0beb5ff
-# linux         armv7l      armv7l          armv7l           false           60           v3.2            fb359370b052a47ce5c84cc6b4a7a03ed7053b25
-linux           32          32              i686             false           60           v3.2            209c4db679a515befd7fb50ecc6bfbecf7ec3d32
-# linux         ppc64le     ppc64le         powerpc64le      false           60           v3.2            c03a0158b19d48ac84b426834fce0d3584cdd0c7
-linux           64          64_rr           x86_64           false           180          v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
-linux           64          64_st           x86_64           false           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
-linux           64          64_mt           x86_64           false           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
-musl            64          64              x86_64           true            60           v3.19           e6a2730e37c386c46915b2650d6aaaa398195152
+# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
+# linux       _aarch64    false         _aarch64    aarch64        none           60         no       no       no       v3.2          0566841e29f0f9880541c26a6595fd5ce0beb5ff
+# linux       _armv7l     false         _armv7l     armv7l         none           60         no       no       no       v3.2          fb359370b052a47ce5c84cc6b4a7a03ed7053b25
+linux         32          false         32          i686           none           60         no       no       no       v3.2          209c4db679a515befd7fb50ecc6bfbecf7ec3d32
+# linux       _ppc64le    false         _ppc64le    powerpc64le    none           60         no       no       no       v3.2          c03a0158b19d48ac84b426834fce0d3584cdd0c7
+linux         64_rr       false         64          x86_64         none           60         yes      no       no       v3.2          474bf61a926b2d7fcf202284d59d4b11a04601d7
+linux         64_st       false         64          x86_64         none           60         no       yes      no       v3.2          474bf61a926b2d7fcf202284d59d4b11a04601d7
+linux         64_mt       false         64          x86_64         none           60         no       no       yes      v3.2          474bf61a926b2d7fcf202284d59d4b11a04601d7
+musl          64          true          64          x86_64         none           60         no       no       no       v3.19         e6a2730e37c386c46915b2650d6aaaa398195152

.buildkite/pipelines/main/platforms/tester_linux.yml

@@ -4,18 +4,18 @@ agents:
   sandbox.jl: "true"
   os: "linux"
 steps:
-  - label: "tester_${PLATFORM?}${ARCH_LABEL?}"
-    key: tester_${PLATFORM?}${ARCH_LABEL?}
+  - label: "tester_${PLATFORM?}${LABEL?}"
+    key: tester_${PLATFORM?}${LABEL?}
     depends_on: package_${PLATFORM?}${ARCH?}
     plugins:
       - JuliaCI/julia#v1:
           # Drop default "registries" directory, so it is not persisted from execution to execution
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz
-          # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz
-          rootfs_treehash: "${ROOTFS_TREE?}"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz
+          # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz
+          rootfs_treehash: "${ROOTFS_HASH?}"
           uid: 1000
           gid: 1000
           workspaces:
@@ -57,7 +57,7 @@ steps:
       export OPENBLAS_NUM_THREADS=8
       export TESTS="[\"all\"]"
 
-      if [[ "$${BUILDKITE_STEP_KEY:?}" == "tester_linux64_rr" ]]; then
+      if [[ "${IS_RR?}" == "yes" ]]; then
         export JULIA_BINARY_UNDER_RR="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
 
         export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY_UNDER_RR:?}"
@@ -69,11 +69,11 @@ steps:
         export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY:?}"
         export NCORES_FOR_TESTS="Sys.CPU_THREADS"
 
-        if [[ "$${BUILDKITE_STEP_KEY:?}"   == "tester_linux64_st" ]]; then
-          # "_st" = single-threaded
+        if [[ "${IS_ST?}"   == "yes" ]]; then
+          # "ST" = single-threaded
           export JULIA_NUM_THREADS=1
-        elif [[ "$${BUILDKITE_STEP_KEY:?}" == "tester_linux64_mt" ]]; then
-          # "_mt" = multi-threaded
+        elif [[ "${IS_MT?}" == "yes" ]]; then
+          # "MT" = multi-threaded
           export JULIA_NUM_THREADS=16
 
           # We do not run the Distributed test suite when multithreading is enabled.

.buildkite/pipelines/scheduled/0_webui.yml

@@ -13,8 +13,8 @@ steps:
           # Build secrets will not be available in these pipelines (or their children)
           # but some of our signed pipelines can wait upon the completion of these unsigned
           # pipelines.
-          # unsigned_pipelines:
-          #   - .buildkite/pipelines/scheduled/launch_unsigned_builders.yml
+          unsigned_pipelines:
+            - .buildkite/pipelines/scheduled/launch_unsigned_jobs.yml
 
           # Our signed pipelines must have a `signature` or `signature_file` parameter that
           # verifies the treehash of the pipeline itself and the inputs listed in `inputs`

.buildkite/pipelines/scheduled/launch_unsigned_jobs.yml

@@ -0,0 +1,8 @@
+steps:
+  - label: ":buildkite: Launch unsigned jobs"
+    commands: |
+      # Launch all of the `USE_BINARYBUILDER=0` jobs.
+      bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml
+      bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml
+    agents:
+      queue: julia

.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches

@@ -0,0 +1,2 @@
+# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS             TIMEOUT    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
+linux         64_no_bb    false         64_no_bb    x86_64         USE_BINARYBUILDER=0    240        no       no       no       v4.4          ec3873fc1756ecb4c815e8785e9ca45cf54c68b6

.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches

@@ -0,0 +1,2 @@
+# PLATFORM    LABEL          ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
+linux         64_st_no_bb    false         64_no_bb    x86_64         none           60         no       yes      no       v4.4          ec3873fc1756ecb4c815e8785e9ca45cf54c68b6

.buildkite/utilities/platforms/platforms.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+ARCHES="$1"
+YAML="$2"
+
+if [[ ! -f "${ARCHES:?}" ]] ; then
+  echo "File does not exist: ${ARCHES:?}"
+  exit 1
+fi
+
+if [[ ! -f "${YAML:?}" ]] ; then
+  echo "File does not exist: ${YAML:?}"
+  exit 1
+fi
+
+cat "${ARCHES:?}" | tr -s ' ' | while read _line; do
+  # Remove whitespace from the beginning and end of each line
+  line=`echo $_line | tr -s ' '`
+
+  # Skip all lines that begin with `#`
+  if [[ $line == \#* ]]; then
+    continue
+  fi
+
+  export PLATFORM=`echo $line    | cut -d ' ' -f 1  | tr -s ' '`
+  export LABEL=`echo $line       | cut -d ' ' -f 2  | tr -s ' '`
+  export ALLOW_FAIL=`echo $line  | cut -d ' ' -f 3  | tr -s ' '`
+  export ARCH=`echo $line        | cut -d ' ' -f 4  | tr -s ' '`
+  export ARCH_ROOTFS=`echo $line | cut -d ' ' -f 5  | tr -s ' '`
+  export MAKE_FLAGS=`echo $line  | cut -d ' ' -f 6  | tr -s ' '`
+  export TIMEOUT=`echo $line     | cut -d ' ' -f 7  | tr -s ' '`
+  export IS_RR=`echo $line       | cut -d ' ' -f 8  | tr -s ' '`
+  export IS_ST=`echo $line       | cut -d ' ' -f 9  | tr -s ' '`
+  export IS_MT=`echo $line       | cut -d ' ' -f 10 | tr -s ' '`
+  export ROOTFS_TAG=`echo $line  | cut -d ' ' -f 11 | tr -s ' '`
+  export ROOTFS_HASH=`echo $line | cut -d ' ' -f 12 | tr -s ' '`
+
+  if [[ "${MAKE_FLAGS:?}" == "none" ]]; then
+    export MAKE_FLAGS=""
+  fi
+
+  buildkite-agent pipeline upload "${YAML:?}"
+done