-
-
Notifications
You must be signed in to change notification settings - Fork 5.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
CI (Buildkite): Add the
package_linux64
and tester_linux64
pipelines
- Loading branch information
1 parent
56bb340
commit 6687f26
Showing
3 changed files
with
215 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones | ||
# since we need nestable sandboxing. The rootfs images being used here are built from | ||
# the `.buildkite/rootfs_images/llvm-passes.jl` file. | ||
agents: | ||
queue: "julia" | ||
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing | ||
sandbox.jl: "true" | ||
os: "linux" | ||
|
||
steps: | ||
- label: "package_linux64" | ||
key: package_linux64 | ||
plugins: | ||
- JuliaCI/julia#v1: | ||
version: 1.6 | ||
- staticfloat/sandbox#v1: | ||
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz | ||
rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" | ||
uid: 1000 | ||
gid: 1000 | ||
workspaces: | ||
# Include `/cache/repos` so that our `git` version introspection works. | ||
- "/cache/repos:/cache/repos" | ||
commands: | | ||
echo "--- Build Julia from source" | ||
make -j 6 | ||
make -j 6 release | ||
make install | ||
echo "--- Print Julia version info" | ||
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' | ||
./julia -e '@info "" Sys.CPU_THREADS' | ||
echo "--- Upload build artifacts" | ||
./julia -e 'println(filter(startswith("julia-"), readdir(pwd())))' | ||
./julia -e 'println(only(filter(startswith("julia-"), readdir(pwd()))))' # make sure there is only a single directory of the form `julia-*`` | ||
ls -d julia-* | ||
mv julia-* julia-release | ||
ls -d julia-* | ||
ls -d julia-release | ||
rm -rf julia-release.tar.gz | ||
tar cfz julia-release.tar.gz julia-release/ | ||
ls julia-release.tar.gz | ||
buildkite-agent artifact upload julia-release.tar.gz | ||
timeout_in_minutes: 60 | ||
notify: | ||
- github_commit_status: | ||
context: "package_linux64" | ||
|
||
- label: "tester_linux64" | ||
key: tester_linux64 | ||
depends_on: package_linux64 # TODO: uncomment this line | ||
plugins: | ||
- JuliaCI/julia#v1: | ||
version: 1.6 | ||
- staticfloat/sandbox#v1: | ||
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz | ||
rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" | ||
uid: 1000 | ||
gid: 1000 | ||
workspaces: | ||
# Include `/cache/repos` so that our `git` version introspection works. | ||
- "/cache/repos:/cache/repos" | ||
commands: | | ||
echo "--- Build Julia from source" | ||
make -j 6 | ||
echo "--- Print Julia version info" | ||
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' | ||
./julia -e '@info "" Sys.CPU_THREADS' | ||
echo "--- Run the Julia test suite" | ||
# TODO: change "abstractarray" to "all" | ||
# TODO: provide the correct value for MY_RUNID | ||
# TODO: provide the correct value for MY_SHORTCOMMIT | ||
./julia .buildkite/rr_capture.jl MY_RUNID MY_SHORTCOMMIT ./julia -e 'Base.runtests(["abstractarray"]; ncores = Sys.CPU_THREADS)' | ||
timeout_in_minutes: 120 | ||
notify: | ||
- github_commit_status: | ||
context: "tester_linux64" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,132 @@ | ||
using Dates | ||
using Pkg | ||
using Tar | ||
|
||
if Base.VERSION < v"1.6" | ||
throw(ErrorException("The `rr_capture.jl` script requires Julia 1.6 or greater")) | ||
end | ||
|
||
if length(ARGS) < 3 | ||
throw(ErrorException("Usage: rr_capture.jl [buildnumber] [shortcommit] [command...]")) | ||
end | ||
|
||
const TIMEOUT = 2 * 60 * 60 # timeout in seconds | ||
|
||
const run_id = popfirst!(ARGS) | ||
const shortcommit = popfirst!(ARGS) | ||
|
||
@info "" run_id shortcommit | ||
@info "" ARGS | ||
|
||
# We only use `rr` on the `tester_linux64` builder. | ||
# if ENV["BUILDKITE_LABEL"] != "tester_linux64" # TODO: uncomment this line | ||
if true # TODO: delete this line | ||
@info "We will not run the tests under rr" | ||
cmd = `$ARGS` | ||
@info "Running command..." cmd | ||
p = run(cmd) | ||
@info "" p.exitcode | ||
exit(p.exitcode) | ||
end | ||
|
||
@info "We will run the tests under rr" | ||
|
||
const num_cores = min(Sys.CPU_THREADS, 8, parse(Int, get(ENV, "JULIA_TEST_NUM_CORES", "8")) + 1) | ||
|
||
proc = nothing | ||
|
||
new_env = copy(ENV) | ||
mktempdir() do dir | ||
Pkg.activate(dir) | ||
Pkg.add("rr_jll") | ||
Pkg.add("Zstd_jll") | ||
|
||
rr_jll = Base.require(Base.PkgId(Base.UUID((0xe86bdf43_55f7_5ea2_9fd0_e7daa2c0f2b4)), "rr_jll")) | ||
zstd_jll = Base.require(Base.PkgId(Base.UUID((0x3161d3a3_bdf6_5164_811a_617609db77b4)), "Zstd_jll")) | ||
rr(func) = Base.invokelatest(rr_jll.rr, func; adjust_LIBPATH=false) | ||
rr() do rr_path | ||
capture_script_path = joinpath(dir, "capture_output.sh") | ||
loader = Sys.WORD_SIZE == 64 ? "/lib64/ld-linux-x86-64.so.2" : "/lib/ld-linux.so.2" | ||
open(capture_script_path, "w") do io | ||
write(io, """ | ||
#!/bin/bash | ||
$(rr_path) record --nested=detach "\$@" > >(tee -a $(dir)/stdout.log) 2> >(tee -a $(dir)/stderr.log >&2) | ||
""") | ||
end | ||
chmod(capture_script_path, 0o755) | ||
|
||
new_env = copy(ENV) | ||
new_env["_RR_TRACE_DIR"] = joinpath(dir, "rr_traces") | ||
new_env["RR_LOG"]="all:debug" | ||
new_env["RR_LOG_BUFFER"]="100000" | ||
new_env["JULIA_RR"] = capture_script_path | ||
t_start = time() | ||
global proc = run(setenv(`$(rr_path) record --num-cores=$(num_cores) $ARGS`, new_env), (stdin, stdout, stderr); wait=false) | ||
|
||
# Start asynchronous timer that will kill `rr` | ||
@async begin | ||
sleep(TIMEOUT) | ||
|
||
# If we've exceeded the timeout and `rr` is still running, kill it. | ||
if isopen(proc) | ||
println(stderr, "\n\nProcess timed out. Signalling `rr` for force-cleanup!") | ||
kill(proc, Base.SIGTERM) | ||
|
||
# Give `rr` a chance to cleanup | ||
sleep(60) | ||
|
||
if isopen(proc) | ||
println(stderr, "\n\n`rr` failed to cleanup within one minute, killing and exiting immediately!") | ||
kill(proc, Base.SIGKILL) | ||
exit(1) | ||
end | ||
end | ||
end | ||
|
||
# Wait for `rr` to finish, either through naturally finishing its run, or `SIGTERM`. | ||
# If we have to `SIGKILL` | ||
wait(proc) | ||
|
||
# On a non-zero exit code, upload the `rr` trace | ||
if !success(proc) | ||
println(stderr, "`rr` returned $(proc.exitcode), packing and uploading traces...") | ||
|
||
if !isdir(joinpath(dir, "rr_traces")) | ||
println(stderr, "No `rr_traces` directory! Did `rr` itself fail?") | ||
exit(1) | ||
end | ||
|
||
# Clean up non-traces | ||
rm(joinpath(dir, "rr_traces", "latest-trace")) | ||
rm(joinpath(dir, "rr_traces", "cpu_lock")) | ||
|
||
# Create a directory for the pack files to go | ||
pack_dir = joinpath(dir, "pack") | ||
mkdir(pack_dir) | ||
|
||
# Pack all traces | ||
trace_dirs = [joinpath(dir, "rr_traces", f) for f in readdir(joinpath(dir, "rr_traces"))] | ||
filter!(isdir, trace_dirs) | ||
run(ignorestatus(`$(rr_path) pack --pack-dir=$pack_dir $(trace_dirs)`)) | ||
|
||
# Tar it up | ||
mkpath("dumps") | ||
datestr = Dates.format(now(), dateformat"yyyy-mm-dd_HH_MM_SS") | ||
dst_path = "dumps/rr-run_$(run_id)-gitsha_$(shortcommit)-$(datestr).tar.zst" | ||
zstd_jll.zstdmt() do zstdp | ||
tarproc = open(`$zstdp -o $dst_path`, "w") | ||
Tar.create(dir, tarproc) | ||
close(tarproc.in) | ||
end | ||
end | ||
end | ||
end | ||
|
||
# Pass the exit code back up to Buildkite | ||
if proc.termsignal != 0 | ||
ccall(:raise, Cvoid, (Cint,), proc.termsignal) | ||
exit(1) # Just in case the signal did not cause an exit | ||
else | ||
exit(proc.exitcode) | ||
end |