-
-
Notifications
You must be signed in to change notification settings - Fork 5.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
CI (Buildkite): Add the
package_linux64
, tester_linux64
, and `doc…
…test` builders
- Loading branch information
1 parent
114ee17
commit 6b3d403
Showing
7 changed files
with
289 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones | ||
# since we need nestable sandboxing. The rootfs images being used here are built from | ||
# the `.buildkite/rootfs_images/llvm-passes.jl` file. | ||
agents: | ||
queue: "julia" | ||
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing | ||
sandbox.jl: "true" | ||
os: "linux" | ||
|
||
steps: | ||
- label: "doctest" | ||
key: doctest | ||
plugins: | ||
- JuliaCI/julia#v1: | ||
version: 1.6 | ||
- staticfloat/sandbox#v1: | ||
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1.1/llvm-passes.tar.gz | ||
rootfs_treehash: "e2ac426534f821efc0194a0cc8d79340c2c5ae23" | ||
uid: 1000 | ||
gid: 1000 | ||
workspaces: | ||
# Include `/cache/repos` so that our `git` version introspection works. | ||
- "/cache/repos:/cache/repos" | ||
commands: | | ||
echo "--- Build Julia from source" | ||
make -j 6 | ||
echo "--- Print Julia version info" | ||
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' | ||
echo "--- Build Julia docs" | ||
make docs | ||
echo "--- Run Julia doctests" | ||
JULIA_NUM_THREADS=1 make -C doc doctest=true | ||
timeout_in_minutes: 120 | ||
notify: | ||
- github_commit_status: | ||
context: "doctest" |
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,106 @@ | ||
# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones | ||
# since we need nestable sandboxing. The rootfs images being used here are built from | ||
# the `.buildkite/rootfs_images/llvm-passes.jl` file. | ||
agents: | ||
queue: "julia" | ||
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing | ||
sandbox.jl: "true" | ||
os: "linux" | ||
|
||
steps: | ||
- label: "package_linux64" | ||
key: package_linux64 | ||
plugins: | ||
- JuliaCI/julia#v1: | ||
version: 1.6 | ||
- staticfloat/sandbox#v1: | ||
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/package_linux64.tar.gz | ||
rootfs_treehash: "6b521782a12c789148465f5270788d9883a06894" | ||
uid: 1000 | ||
gid: 1000 | ||
workspaces: | ||
# Include `/cache/repos` so that our `git` version introspection works. | ||
- "/cache/repos:/cache/repos" | ||
commands: | | ||
echo "--- Build Julia from source" | ||
make -j 6 | ||
make release | ||
make install | ||
echo "--- Print Julia version info" | ||
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' | ||
echo "--- Compress build artifacts" | ||
mv julia-* julia-artifact | ||
rm -rf julia-linux64.tar.gz | ||
tar czf julia-linux64.tar.gz julia-artifact/ | ||
echo "--- Upload build artifacts" | ||
buildkite-agent artifact upload julia-linux64.tar.gz | ||
timeout_in_minutes: 60 | ||
notify: | ||
- github_commit_status: | ||
context: "package_linux64" | ||
|
||
- label: "tester_linux64" | ||
key: tester_linux64 | ||
depends_on: package_linux64 | ||
plugins: | ||
- JuliaCI/julia#v1: | ||
version: 1.6 | ||
- staticfloat/sandbox#v1: | ||
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/package_linux64.tar.gz | ||
rootfs_treehash: "6b521782a12c789148465f5270788d9883a06894" | ||
uid: 1000 | ||
gid: 1000 | ||
workspaces: | ||
# Include `/cache/repos` so that our `git` version introspection works. | ||
- "/cache/repos:/cache/repos" | ||
env: | ||
JULIA_SHELL: "/bin/bash" | ||
commands: | | ||
echo "--- Debugging: echo USER" | ||
echo $$USER | ||
echo "--- Debugging: echo HOME" | ||
echo $$HOME | ||
echo "--- Debugging: id" | ||
id | ||
echo "--- Debugging: which bash" | ||
which bash | ||
echo "--- Debugging: which -a bash" | ||
which -a bash | ||
echo "--- Debugging: SHELL" | ||
echo $$SHELL | ||
echo "--- Debugging: JULIA_SHELL" | ||
echo $$JULIA_SHELL | ||
echo "--- Debugging: echo HOME" | ||
echo $$HOME | ||
echo "--- Debugging: ls -ld HOME" | ||
ls -ld $$HOME | ||
echo "--- Download build artifacts" | ||
rm -rf julia-linux64.tar.gz | ||
buildkite-agent artifact download julia-linux64.tar.gz . | ||
echo "--- Extract build artifacts" | ||
rm -rf julia-artifact/ | ||
tar xzf julia-linux64.tar.gz julia-artifact/ | ||
echo "--- Print Julia version info" | ||
julia-artifact/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' | ||
julia-artifact/bin/julia -e '@info "" Sys.CPU_THREADS homedir() withenv(homedir, "HOME" => nothing) ENV["USER"]' | ||
echo "--- Run the Julia test suite" | ||
julia-artifact/bin/julia .buildkite/rr_capture.jl julia-artifact/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)' | ||
timeout_in_minutes: 120 | ||
notify: | ||
- github_commit_status: | ||
context: "tester_linux64" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,134 @@ | ||
using Dates | ||
using Pkg | ||
using Tar | ||
|
||
if Base.VERSION < v"1.6" | ||
throw(ErrorException("The `rr_capture.jl` script requires Julia 1.6 or greater")) | ||
end | ||
|
||
if length(ARGS) < 1 | ||
throw(ErrorException("Usage: rr_capture.jl [command...]")) | ||
end | ||
|
||
const TIMEOUT = 2 * 60 * 60 # timeout in seconds | ||
|
||
# We only use `rr` on the `tester_linux64` builder | ||
const use_rr_if_builder_is = "tester_linux64" | ||
|
||
const run_id = get(ENV, "BUILDKITE_JOB_ID", "unknown") | ||
const shortcommit = get(ENV, "BUILDKITE_COMMIT", "unknown") | ||
const builder = get(ENV, "BUILDKITE_STEP_KEY", use_rr_if_builder_is) | ||
const use_rr = builder == use_rr_if_builder_is | ||
|
||
@info "" run_id shortcommit builder use_rr | ||
@info "" ARGS | ||
|
||
# if !use_rr # TODO: uncomment this line | ||
if true # TODO: delete this line | ||
@info "We will not run the tests under rr" | ||
p = run(`$ARGS`) | ||
exit(p.exitcode) | ||
end | ||
|
||
@info "We will run the tests under rr" | ||
|
||
const num_cores = min(Sys.CPU_THREADS, 8, parse(Int, get(ENV, "JULIA_TEST_NUM_CORES", "8")) + 1) | ||
@info "" num_cores | ||
|
||
proc = nothing | ||
|
||
new_env = copy(ENV) | ||
mktempdir() do dir | ||
Pkg.activate(dir) | ||
Pkg.add("rr_jll") | ||
Pkg.add("Zstd_jll") | ||
|
||
rr_jll = Base.require(Base.PkgId(Base.UUID((0xe86bdf43_55f7_5ea2_9fd0_e7daa2c0f2b4)), "rr_jll")) | ||
zstd_jll = Base.require(Base.PkgId(Base.UUID((0x3161d3a3_bdf6_5164_811a_617609db77b4)), "Zstd_jll")) | ||
rr(func) = Base.invokelatest(rr_jll.rr, func; adjust_LIBPATH=false) | ||
rr() do rr_path | ||
capture_script_path = joinpath(dir, "capture_output.sh") | ||
loader = Sys.WORD_SIZE == 64 ? "/lib64/ld-linux-x86-64.so.2" : "/lib/ld-linux.so.2" | ||
open(capture_script_path, "w") do io | ||
write(io, """ | ||
#!/bin/bash | ||
$(rr_path) record --nested=detach "\$@" > >(tee -a $(dir)/stdout.log) 2> >(tee -a $(dir)/stderr.log >&2) | ||
""") | ||
end | ||
chmod(capture_script_path, 0o755) | ||
|
||
new_env = copy(ENV) | ||
new_env["_RR_TRACE_DIR"] = joinpath(dir, "rr_traces") | ||
new_env["RR_LOG"]="all:debug" | ||
new_env["RR_LOG_BUFFER"]="100000" | ||
new_env["JULIA_RR"] = capture_script_path | ||
t_start = time() | ||
global proc = run(setenv(`$(rr_path) record --num-cores=$(num_cores) $ARGS`, new_env), (stdin, stdout, stderr); wait=false) | ||
|
||
# Start asynchronous timer that will kill `rr` | ||
@async begin | ||
sleep(TIMEOUT) | ||
|
||
# If we've exceeded the timeout and `rr` is still running, kill it. | ||
if isopen(proc) | ||
println(stderr, "\n\nProcess timed out. Signalling `rr` for force-cleanup!") | ||
kill(proc, Base.SIGTERM) | ||
|
||
# Give `rr` a chance to cleanup | ||
sleep(60) | ||
|
||
if isopen(proc) | ||
println(stderr, "\n\n`rr` failed to cleanup within one minute, killing and exiting immediately!") | ||
kill(proc, Base.SIGKILL) | ||
exit(1) | ||
end | ||
end | ||
end | ||
|
||
# Wait for `rr` to finish, either through naturally finishing its run, or `SIGTERM`. | ||
# If we have to `SIGKILL` | ||
wait(proc) | ||
|
||
# On a non-zero exit code, upload the `rr` trace | ||
if !success(proc) | ||
println(stderr, "`rr` returned $(proc.exitcode), packing and uploading traces...") | ||
|
||
if !isdir(joinpath(dir, "rr_traces")) | ||
println(stderr, "No `rr_traces` directory! Did `rr` itself fail?") | ||
exit(1) | ||
end | ||
|
||
# Clean up non-traces | ||
rm(joinpath(dir, "rr_traces", "latest-trace")) | ||
rm(joinpath(dir, "rr_traces", "cpu_lock")) | ||
|
||
# Create a directory for the pack files to go | ||
pack_dir = joinpath(dir, "pack") | ||
mkdir(pack_dir) | ||
|
||
# Pack all traces | ||
trace_dirs = [joinpath(dir, "rr_traces", f) for f in readdir(joinpath(dir, "rr_traces"))] | ||
filter!(isdir, trace_dirs) | ||
run(ignorestatus(`$(rr_path) pack --pack-dir=$pack_dir $(trace_dirs)`)) | ||
|
||
# Tar it up | ||
mkpath("dumps") | ||
datestr = Dates.format(now(), dateformat"yyyy-mm-dd_HH_MM_SS") | ||
dst_path = "dumps/rr-run_$(run_id)-gitsha_$(shortcommit)-$(datestr).tar.zst" | ||
zstd_jll.zstdmt() do zstdp | ||
tarproc = open(`$zstdp -o $dst_path`, "w") | ||
Tar.create(dir, tarproc) | ||
close(tarproc.in) | ||
end | ||
end | ||
end | ||
end | ||
|
||
# Pass the exit code back up to Buildkite | ||
if proc.termsignal != 0 | ||
ccall(:raise, Cvoid, (Cint,), proc.termsignal) | ||
exit(1) # Just in case the signal did not cause an exit | ||
else | ||
exit(proc.exitcode) | ||
end |