add support for daily benchmark jobs

README.md

@@ -60,6 +60,18 @@ The `vs` keyword argument takes a reference string which can points to a Julia c
 - `"owner/repo:branch"`: the head commit of the branch named `branch` in the repository `owner/repo`
 - `"owner/repo@sha"`: the commit specified by `sha` in the repository `owner/repo`
 
+##### Benchmark Results
+
+Once a `BenchmarjJob` is complete, the results are uploaded to the
+[BaseBenchmarkReports](https://github.com/JuliaCI/BaseBenchmarkReports) repository. Each job
+has its own directory for results. This directory contains the following items:
+
+- `report.md` is a markdown report that summarizes the job results
+- `data.tar.gz` contains raw timing data in JLD format. To untar this file, run
+`tar -xzvf data.tar.gz`. You can analyze this data using the
+[BenchmarkTools](https://github.com/JuliaCI/BaseBenchmarkReports) package.
+- `logs` is a directory containing the build logs and benchmark execution logs for the job.
+
 ##### Comment Examples
 
 Here are some examples of comments that trigger a `BenchmarkJob` in various contexts:

bin/submit_daily_job.jl

@@ -0,0 +1,12 @@
+import GitHub
+
+auth = GitHub.authenticate(ENV["GITHUB_AUTH"])
+repo = "JuliaLang/julia"
+sha = get(get(GitHub.branch(repo, "master").commit).sha)
+message = """
+          Executing the daily benchmark build, I will reply here when finished:
+
+          @nanosoldier `runbenchmarks(ALL, isdaily = true)`
+          """
+
+GitHub.create_comment(repo, sha, :commit, auth = auth, params = Dict("body" => message))

src/Nanosoldier.jl

@@ -11,6 +11,11 @@ const BRANCH_SEPARATOR = ':'
 snip(str, len) = length(str) > len ? str[1:len] : str
 snipsha(sha) = snip(sha, 7)
 
+gitclone!(repo, path) = run(`git clone git@github.com:$(repo).git $(path)`)
+
+gitreset!() = (run(`git fetch --all`); run(`git reset --hard origin/master`))
+gitreset!(path) = cd(gitreset!, path)
+
 include("config.jl")
 include("build.jl")
 include("submission.jl")

src/build.jl

@@ -19,7 +19,7 @@ end
 Base.summary(build::BuildRef) = string(build.repo, SHA_SEPARATOR, snipsha(build.sha))
 
 # if a PR number is included, attempt to build from the PR's merge commit
-function build_julia!(config::Config, build::BuildRef, prnumber::Nullable{Int} = Nullable{Int}())
+function build_julia!(config::Config, build::BuildRef, logpath, prnumber::Nullable{Int} = Nullable{Int}())
     # make a temporary workdir for our build
     builddir = mktempdir(workdir(config))
     cd(workdir(config))
@@ -28,7 +28,7 @@ function build_julia!(config::Config, build::BuildRef, prnumber::Nullable{Int} =
     if !(isnull(prnumber))
         pr = get(prnumber)
         # clone from `trackrepo`, not `build.repo`, since that's where the merge commit is
-        run(`git clone --quiet https://github.com/$(config.trackrepo) $(builddir)`)
+        gitclone!(config.trackrepo, builddir)
         cd(builddir)
         try
             run(`git fetch --quiet origin +refs/pull/$(pr)/merge:`)
@@ -40,15 +40,15 @@ function build_julia!(config::Config, build::BuildRef, prnumber::Nullable{Int} =
         run(`git checkout --quiet --force FETCH_HEAD`)
         build.sha = readchomp(`git rev-parse HEAD`)
     else
-        run(`git clone --quiet https://github.com/$(build.repo) $(builddir)`)
+        gitclone!(build.repo, builddir)
         cd(builddir)
         run(`git checkout --quiet $(build.sha)`)
     end
 
     # set up logs for STDOUT and STDERR
     logname = string(build.sha, "_build")
-    outfile = joinpath(logdir(config), string(logname, ".out"))
-    errfile = joinpath(logdir(config), string(logname, ".err"))
+    outfile = joinpath(logpath, string(logname, ".out"))
+    errfile = joinpath(logpath, string(logname, ".err"))
 
     # run the build
     run(pipeline(`make`, stdout = outfile, stderr = errfile))

src/config.jl

@@ -24,24 +24,26 @@ end
 # the shared space in which child nodes can work
 workdir(config::Config) = config.workdir
 
-# the directory where results are stored
-resultdir(config::Config) = joinpath(workdir(config), "results")
+# the report repository
+reportrepo(config::Config) = config.reportrepo
 
-# the directory where build logs are stored
-logdir(config::Config) = joinpath(workdir(config), "logs")
+# the local directory of the report repository
+reportdir(config::Config) = joinpath(workdir(config), split(reportrepo(config), "/")[2])
 
-# ensure directories exists
 persistdir!(path) = (!(isdir(path)) && mkdir(path); return path)
 
 function persistdir!(config::Config)
     persistdir!(workdir(config))
-    persistdir!(resultdir(config))
-    persistdir!(logdir(config))
+    if isdir(reportdir(config))
+        gitreset!(reportdir(config))
+    else
+        gitclone!(reportrepo(config), reportdir(config))
+    end
 end
 
 function nodelog(config::Config, node, message)
-    persistdir!(logdir(config))
-    open(joinpath(logdir(config), "node$(node).log"), "a") do file
+    persistdir!(workdir(config))
+    open(joinpath(workdir(config), "node$(node).log"), "a") do file
         println(file, now(), " | ", node, " | ", message)
     end
 end