From 23fd0552060613d971a957a007cd4cc8a346d518 Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Sun, 5 Jun 2016 13:37:40 -0400
Subject: [PATCH 01/13] add daily option and better phrase match validation
 testing

---
 src/jobs/BenchmarkJob.jl | 21 +++++++++++++++-----
 src/submission.jl        |  1 +
 test/runtests.jl         | 42 +++++++++++++++++++++++++++-------------
 3 files changed, 46 insertions(+), 18 deletions(-)

diff --git a/src/jobs/BenchmarkJob.jl b/src/jobs/BenchmarkJob.jl
index fb7929c..edf12a1 100644
--- a/src/jobs/BenchmarkJob.jl
+++ b/src/jobs/BenchmarkJob.jl
@@ -44,7 +44,8 @@ type BenchmarkJob <: AbstractJob
     submission::JobSubmission
     tagpred::UTF8String         # predicate string to be fed to @tagged
     against::Nullable{BuildRef} # the comparison build (if available)
-    skipbuild::Bool
+    skipbuild::Bool             # use local v0.5 install instead of a fresh build (for testing)
+    daily::Bool                 # the submitted job is a daily build
 end
 
 function BenchmarkJob(submission::JobSubmission)
@@ -65,12 +66,20 @@ function BenchmarkJob(submission::JobSubmission)
     else
         against = Nullable{BuildRef}()
     end
+
     if haskey(submission.kwargs, :skipbuild)
         skipbuild = submission.kwargs[:skipbuild] == "true"
     else
         skipbuild = false
     end
-    return BenchmarkJob(submission, first(submission.args), against, skipbuild)
+
+    if haskey(submission.kwargs, :daily)
+        daily = submission.kwargs[:daily] == "true"
+    else
+        daily = false
+    end
+
+    return BenchmarkJob(submission, first(submission.args), against, skipbuild, daily)
 end
 
 function branchref(config::Config, reponame::AbstractString, branchname::AbstractString)
@@ -87,10 +96,12 @@ function Base.summary(job::BenchmarkJob)
 end
 
 function isvalid(submission::JobSubmission, ::Type{BenchmarkJob})
+    allowed_kwargs = (:vs, :skipbuild, :daily)
     args, kwargs = submission.args, submission.kwargs
-    return (submission.func == "runbenchmarks" &&
-            (length(args) == 1 && is_valid_tagpred(first(args))) &&
-            (isempty(kwargs) || (length(kwargs) < 3)))
+    has_valid_args = length(args) == 1 && is_valid_tagpred(first(args))
+    has_valid_kwargs = (all(key -> in(key, allowed_kwargs), keys(kwargs)) &&
+                        (length(kwargs) <= length(allowed_kwargs)))
+    return (submission.func == "runbenchmarks") && has_valid_args && has_valid_kwargs
 end
 
 submission(job::BenchmarkJob) = job.submission
diff --git a/src/submission.jl b/src/submission.jl
index 161535e..4567fb5 100644
--- a/src/submission.jl
+++ b/src/submission.jl
@@ -97,6 +97,7 @@ function parse_phrase_match(phrase_match::AbstractString)
         started_kwargs = false
         for x in parsed_args.args
             if isa(x, Expr) && (x.head == :kw || x.head == :(=)) && isa(x.args[1], Symbol)
+                @assert !(haskey(kwargs, x.args[1])) "kwargs must all be unique"
                 kwargs[x.args[1]] = phrase_argument(x.args[2])
                 started_kwargs = true
             else
diff --git a/test/runtests.jl b/test/runtests.jl
index e8edd78..965923c 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -3,9 +3,9 @@ using Nanosoldier, Base.Test, Compat, BenchmarkTools
 using Nanosoldier: BuildRef, JobSubmission, Config, BenchmarkJob
 using BenchmarkTools: TrialEstimate, Parameters
 
-##############################
-# Markdown Report Generation #
-##############################
+#########
+# setup #
+#########
 
 vinfo = """
 Julia Version 0.4.3-pre+6
@@ -34,24 +34,40 @@ Intel(R) Core(TM) i5-4288U CPU @ 2.60GHz:
 primary = BuildRef("jrevels/julia", "25c3659d6cec2ebf6e6c7d16b03adac76a47b42a", vinfo)
 against = Nullable(BuildRef("JuliaLang/julia", "bb73f3489d837e3339fce2c1aab283d3b2e97a4c", vinfo*"_against"))
 config = Config("user", [1], [1], GitHub.AnonymousAuth(), "test");
+tagpred = "ALL && !(\"tag1\" || \"tag2\")"
 
-function build_test_submission(tagpred; vs = "")
-    if isempty(vs)
-        phrase_match = "@nanosoldier `runbenchmarks($(tagpred))`"
-    else
-        phrase_match = "@nanosoldier `runbenchmarks($(tagpred); vs = $(vs))`"
-    end
+#####################################
+# submission parsing and validation #
+#####################################
+
+function build_test_submission(phrase_match)
     func, args, kwargs = Nanosoldier.parse_phrase_match(phrase_match)
     submission = JobSubmission(config, primary, primary.sha, "https://www.test.com", :commit, Nullable{Int}(), func, args, kwargs)
     @test Nanosoldier.isvalid(submission, BenchmarkJob)
     return submission
 end
 
-build_test_submission("ALL", vs = "\"JuliaLang/julia:master\"")
-build_test_submission("\"tag\"")
+build_test_submission("@nanosoldier `runbenchmarks(ALL)`")
+build_test_submission("@nanosoldier `runbenchmarks(\"tag\")`")
+build_test_submission("@nanosoldier `runbenchmarks($tagpred)`")
 
-tagpred = "ALL && !(\"tag1\" || \"tag2\")"
-sub = build_test_submission(tagpred)
+build_test_submission("@nanosoldier `runbenchmarks(ALL, vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks(\"tag\", vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks($tagpred, vs = \"JuliaLang/julia:master\")`")
+
+build_test_submission("@nanosoldier `runbenchmarks(ALL, daily = true, vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks(\"tag\", daily = true, vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks($tagpred, daily = true, vs = \"JuliaLang/julia:master\")`")
+
+build_test_submission("@nanosoldier `runbenchmarks(ALL; daily = true, vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks(\"tag\"; daily = true, vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks($tagpred; daily = true, vs = \"JuliaLang/julia:master\")`")
+
+#########################
+# job report generation #
+#########################
+
+sub = build_test_submission("@nanosoldier `runbenchmarks($tagpred)`")
 job = BenchmarkJob(sub)
 @test Nanosoldier.submission(job) == sub
 @test job.tagpred == tagpred

From a8dadb3ff5194e39d6e6caab71dcbc5a5b9089e7 Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Mon, 6 Jun 2016 11:24:22 -0400
Subject: [PATCH 02/13] upload tarred result data and benchmark logs along with
 markdown report

---
 src/Nanosoldier.jl       |  5 +++
 src/build.jl             |  4 +-
 src/config.jl            | 14 ++++--
 src/jobs/BenchmarkJob.jl | 94 ++++++++++++++++++++++------------------
 src/jobs/jobs.jl         |  2 +-
 src/submission.jl        | 22 +++++-----
 test/runtests.jl         | 12 ++---
 7 files changed, 86 insertions(+), 67 deletions(-)

diff --git a/src/Nanosoldier.jl b/src/Nanosoldier.jl
index 87bdd17..dbc5b03 100644
--- a/src/Nanosoldier.jl
+++ b/src/Nanosoldier.jl
@@ -11,6 +11,11 @@ const BRANCH_SEPARATOR = ':'
 snip(str, len) = length(str) > len ? str[1:len] : str
 snipsha(sha) = snip(sha, 7)
 
+gitclone!(repo, path) = run(`git clone https://github.com/$(repo) $(path)`)
+
+gitreset!() = run(`git fetch --all && git reset --hard origin/master`)
+gitreset!(path) = cd(gitreset!, path)
+
 include("config.jl")
 include("build.jl")
 include("submission.jl")
diff --git a/src/build.jl b/src/build.jl
index 9aa0353..c29b90d 100644
--- a/src/build.jl
+++ b/src/build.jl
@@ -28,7 +28,7 @@ function build_julia!(config::Config, build::BuildRef, prnumber::Nullable{Int} =
     if !(isnull(prnumber))
         pr = get(prnumber)
         # clone from `trackrepo`, not `build.repo`, since that's where the merge commit is
-        run(`git clone --quiet https://github.com/$(config.trackrepo) $(builddir)`)
+        gitclone!(config.trackrepo, builddir)
         cd(builddir)
         try
             run(`git fetch --quiet origin +refs/pull/$(pr)/merge:`)
@@ -40,7 +40,7 @@ function build_julia!(config::Config, build::BuildRef, prnumber::Nullable{Int} =
         run(`git checkout --quiet --force FETCH_HEAD`)
         build.sha = readchomp(`git rev-parse HEAD`)
     else
-        run(`git clone --quiet https://github.com/$(build.repo) $(builddir)`)
+        gitclone!(build.repo, builddir)
         cd(builddir)
         run(`git checkout --quiet $(build.sha)`)
     end
diff --git a/src/config.jl b/src/config.jl
index effab7f..2c96cac 100644
--- a/src/config.jl
+++ b/src/config.jl
@@ -24,19 +24,25 @@ end
 # the shared space in which child nodes can work
 workdir(config::Config) = config.workdir
 
-# the directory where results are stored
-resultdir(config::Config) = joinpath(workdir(config), "results")
+# the report repository
+reportrepo(config::Config) = config.reportrepo
+
+# the local directory of the report repository
+reportdir(config::Config) = joinpath(workdir(config), split(reportrepo(config), "/")[2])
 
 # the directory where build logs are stored
 logdir(config::Config) = joinpath(workdir(config), "logs")
 
-# ensure directories exists
 persistdir!(path) = (!(isdir(path)) && mkdir(path); return path)
 
 function persistdir!(config::Config)
     persistdir!(workdir(config))
-    persistdir!(resultdir(config))
     persistdir!(logdir(config))
+    if isdir(reportdir(config))
+        gitreset!(reportdir(config))
+    else
+        gitclone!(reportrepo(config), reportdir(config))
+    end
 end
 
 function nodelog(config::Config, node, message)
diff --git a/src/jobs/BenchmarkJob.jl b/src/jobs/BenchmarkJob.jl
index edf12a1..6a26055 100644
--- a/src/jobs/BenchmarkJob.jl
+++ b/src/jobs/BenchmarkJob.jl
@@ -41,11 +41,12 @@ end
 ################
 
 type BenchmarkJob <: AbstractJob
-    submission::JobSubmission
+    submission::JobSubmission   # the original submission
     tagpred::UTF8String         # predicate string to be fed to @tagged
     against::Nullable{BuildRef} # the comparison build (if available)
+    date::Dates.Date            # the date of the submitted job
+    isdaily::Bool               # is the job a daily job?
     skipbuild::Bool             # use local v0.5 install instead of a fresh build (for testing)
-    daily::Bool                 # the submitted job is a daily build
 end
 
 function BenchmarkJob(submission::JobSubmission)
@@ -73,13 +74,14 @@ function BenchmarkJob(submission::JobSubmission)
         skipbuild = false
     end
 
-    if haskey(submission.kwargs, :daily)
-        daily = submission.kwargs[:daily] == "true"
+    if haskey(submission.kwargs, :isdaily)
+        isdaily = submission.kwargs[:isdaily] == "true"
     else
-        daily = false
+        isdaily = false
     end
 
-    return BenchmarkJob(submission, first(submission.args), against, skipbuild, daily)
+    return BenchmarkJob(submission, first(submission.args), against,
+                        Dates.today(), skipbuild, isdaily)
 end
 
 function branchref(config::Config, reponame::AbstractString, branchname::AbstractString)
@@ -96,7 +98,7 @@ function Base.summary(job::BenchmarkJob)
 end
 
 function isvalid(submission::JobSubmission, ::Type{BenchmarkJob})
-    allowed_kwargs = (:vs, :skipbuild, :daily)
+    allowed_kwargs = (:vs, :skipbuild, :isdaily)
     args, kwargs = submission.args, submission.kwargs
     has_valid_args = length(args) == 1 && is_valid_tagpred(first(args))
     has_valid_kwargs = (all(key -> in(key, allowed_kwargs), keys(kwargs)) &&
@@ -106,6 +108,23 @@ end
 
 submission(job::BenchmarkJob) = job.submission
 
+function jobdirname(job::BenchmarkJob)
+    if job.isdaily
+        return string("daily_", year(job.date), "_", month(job.date), "_", day(job.date))
+    else
+        primarysha = snipsha(submission(job).build.sha)
+        if isnull(job.against)
+            return primarysha
+        else
+            againstsha = snipsha(get(job.against).sha)
+            return string(primarysha, "_vs_", againstsha)
+        end
+    end
+end
+
+reportdir(job::BenchmarkJob) = joinpath(reportdir(submission(job).config), jobdirname(job))
+datadir(job::BenchmarkJob) = joinpath(reportdir(job), "data")
+
 ##########################
 # BenchmarkJob Execution #
 ##########################
@@ -125,6 +144,9 @@ function Base.run(job::BenchmarkJob)
     end
     cd(oldpwd)
 
+    # make data directory for job
+    mkdir(datadir(job))
+
     # run primary job
     nodelog(cfg, node, "running primary build for $(summary(job))")
     primary_results = execute_benchmarks!(job, :primary)
@@ -137,7 +159,7 @@ function Base.run(job::BenchmarkJob)
         against_results = execute_benchmarks!(job, :against)
         nodelog(cfg, node, "finished comparison build for $(summary(job))")
         results["against"] = against_results
-        results["judged"] = BenchmarkTools.judge(primary_results, against_results)
+        results["judged"] = BenchmarkTools.judge(minimum(primary_results), minimum(against_results))
     end
 
     # report results
@@ -201,13 +223,13 @@ function execute_benchmarks!(job::BenchmarkJob, whichbuild::Symbol)
     end
 
     benchname = string(build.sha, "_", whichbuild)
-    benchout = joinpath(logdir(cfg),  string(benchname, ".out"))
-    bencherr = joinpath(logdir(cfg),  string(benchname, ".err"))
-    benchresults = joinpath(resultdir(cfg), string(benchname, ".jld"))
+    benchout = joinpath(reportdir(job), string(benchname, ".out"))
+    bencherr = joinpath(reportdir(job), string(benchname, ".err"))
+    benchresults = joinpath(datadir(job), string(benchname, ".jld"))
 
     open(jlscriptpath, "w") do file
         println(file, """
-                      println(now(), " | starting benchscript.jl (STDOUT/STDERR will be redirected to the logs folder)")
+                      println(now(), " | starting benchscript.jl (STDOUT/STDERR will be redirected to the result folder)")
                       benchout = open(\"$(benchout)\", "w"); redirect_stdout(benchout)
                       bencherr = open(\"$(bencherr)\", "w"); redirect_stderr(bencherr)
 
@@ -231,7 +253,7 @@ function execute_benchmarks!(job::BenchmarkJob, whichbuild::Symbol)
                       warmup(benchmarks)
 
                       println("RUNNING BENCHMARKS...")
-                      results = minimum(run(benchmarks; verbose = true))
+                      results = run(benchmarks; verbose = true)
 
                       println("SAVING RESULT...")
                       JLD.save(\"$(benchresults)\", "results", results)
@@ -287,25 +309,10 @@ end
 function report(job::BenchmarkJob, results)
     node = myid()
     cfg = submission(job).config
-    target_url = ""
     if isempty(results["primary"])
         reply_status(job, "error", "no benchmarks were executed")
         reply_comment(job, "[Your benchmark job]($(submission(job).url)) has completed, but no benchmarks were actually executed. Perhaps your tag predicate contains mispelled tags? cc @jrevels")
     else
-        # To upload our JLD file, we'd need to use the Git Data API, which allows uploading
-        # of large binary blobs. Unfortunately, GitHub.jl doesn't yet implement the Git Data
-        # API, so we don't yet do this. The old code here uploaded a JSON file, but
-        # unfortunately didn't work very consistently because the JSON was often over the
-        # size limit.
-        # try
-        #     datapath = joinpath(reportdir(job), "$(reportfile(job)).json")
-        #     datastr = base64encode(JSON.json(results))
-        #     target_url = upload_report_file(job, datapath, datastr, "upload result data for $(summary(job))")
-        #     nodelog(cfg, node, "uploaded $(datapath) to $(cfg.reportrepo)")
-        # catch err
-        #     nodelog(cfg, node, "error when uploading result JSON file: $(err)")
-        # end
-
         # determine the job's final status
         if !(isnull(job.against))
             found_regressions = BenchmarkTools.isregression(results["judged"])
@@ -316,14 +323,25 @@ function report(job::BenchmarkJob, results)
             status = "successfully executed benchmarks"
         end
 
-        # upload markdown report to the report repository
+        # push to report repo
+        target_url = ""
         try
-            reportpath = joinpath(reportdir(job), "$(reportfile(job)).md")
-            reportstr = base64encode(sprint(io -> printreport(io, job, results)))
-            target_url = upload_report_file(job, reportpath, reportstr, "upload markdown report for $(summary(job))")
-            nodelog(cfg, node, "uploaded $(reportpath) to $(cfg.reportrepo)")
+            # prepare the benchmark data for uploading
+            nodelog(cfg, node, "...preparing data...")
+            cd(reportdir(job)) do
+                run(`tar -zcvf data.tar.gz data`)
+                rm(datadir(job), recursive = true)
+            end
+            # write the markdown report
+            nodelog(cfg, node, "...generating report...")
+            reportname = "report.md"
+            open(joinpath(reportdir(job), reportname)) do file
+                printreport(file, job, results)
+            end
+            # push changes to report repo
+            target_url = upload_report_repo!(job, joinpath(jobdirname(job), reportname), "upload markdown report for $(summary(job))")
         catch err
-            nodelog(cfg, node, "error when uploading markdown report: $(err)")
+            nodelog(cfg, node, "error when pushing to report repo: $(err)")
         end
 
         # reply with the job's final status
@@ -337,14 +355,6 @@ function report(job::BenchmarkJob, results)
     end
 end
 
-reportdir(job::BenchmarkJob) = snipsha(submission(job).build.sha)
-
-function reportfile(job::BenchmarkJob)
-    dir = reportdir(job)
-    return isnull(job.against) ? dir : "$(dir)_vs_$(snipsha(get(job.against).sha))"
-end
-
-
 # Markdown Report Generation #
 #----------------------------#
 
diff --git a/src/jobs/jobs.jl b/src/jobs/jobs.jl
index 0b97642..ba0c653 100644
--- a/src/jobs/jobs.jl
+++ b/src/jobs/jobs.jl
@@ -10,7 +10,7 @@ abstract AbstractJob
 
 reply_status(job::AbstractJob, args...; kwargs...) = reply_status(submission(job), args...; kwargs...)
 reply_comment(job::AbstractJob, args...; kwargs...) = reply_comment(submission(job), args...; kwargs...)
-upload_report_file(job::AbstractJob, args...; kwargs...) = upload_report_file(submission(job), args...; kwargs...)
+upload_report_repo!(job::AbstractJob, args...; kwargs...) = upload_report_repo!(submission(job), args...; kwargs...)
 
 include("BenchmarkJob.jl")
 include("PkgEvalJob.jl")
diff --git a/src/submission.jl b/src/submission.jl
index 4567fb5..22d1c30 100644
--- a/src/submission.jl
+++ b/src/submission.jl
@@ -127,18 +127,16 @@ function reply_comment(sub::JobSubmission, message::AbstractString)
                                  auth = sub.config.auth, params = Dict("body" => message))
 end
 
-function upload_report_file(sub::JobSubmission, path, content, message)
+function upload_report_repo!(sub::JobSubmission, markdownpath, message)
     cfg = sub.config
-    params = Dict("content" => content, "message" => message)
-    # An HTTP response code of 400 means the file doesn't exist, which will cause the
-    # returned `GitHub.Content` object to contain a null `sha` field. We set `handle_error`
-    # to false so that GitHub.jl doesn't throw an error in the case of a 400 response code.
-    priorfile = GitHub.file(cfg.reportrepo, path; auth = cfg.auth, handle_error = false)
-    if isnull(priorfile.sha)
-        results = GitHub.create_file(cfg.reportrepo, path; auth = cfg.auth, params = params)
-    else
-        params["sha"] = get(priorfile.sha)
-        results = GitHub.update_file(cfg.reportrepo, path; auth = cfg.auth, params = params)
+    sha = cd(reportdir(cfg)) do
+        run(`git add -A`)
+        run(`git commit -m $message`)
+        headsha = chomp(readstring(`git rev-parse HEAD`))
+        run(`git pull -X ours`)
+        run(`git push`)
+        return headsha
     end
-    return string(GitHub.permalink(results["content"], results["commit"]))
+    reportfile = GitHub.file(reportrepo(cfg), markdownpath; auth = cfg.auth)
+    return string(GitHub.permalink(reportfile, sha))
 end
diff --git a/test/runtests.jl b/test/runtests.jl
index 965923c..36021d1 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -55,13 +55,13 @@ build_test_submission("@nanosoldier `runbenchmarks(ALL, vs = \"JuliaLang/julia:m
 build_test_submission("@nanosoldier `runbenchmarks(\"tag\", vs = \"JuliaLang/julia:master\")`")
 build_test_submission("@nanosoldier `runbenchmarks($tagpred, vs = \"JuliaLang/julia:master\")`")
 
-build_test_submission("@nanosoldier `runbenchmarks(ALL, daily = true, vs = \"JuliaLang/julia:master\")`")
-build_test_submission("@nanosoldier `runbenchmarks(\"tag\", daily = true, vs = \"JuliaLang/julia:master\")`")
-build_test_submission("@nanosoldier `runbenchmarks($tagpred, daily = true, vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks(ALL, isdaily = true, vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks(\"tag\", isdaily = true, vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks($tagpred, isdaily = true, vs = \"JuliaLang/julia:master\")`")
 
-build_test_submission("@nanosoldier `runbenchmarks(ALL; daily = true, vs = \"JuliaLang/julia:master\")`")
-build_test_submission("@nanosoldier `runbenchmarks(\"tag\"; daily = true, vs = \"JuliaLang/julia:master\")`")
-build_test_submission("@nanosoldier `runbenchmarks($tagpred; daily = true, vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks(ALL; isdaily = true, vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks(\"tag\"; isdaily = true, vs = \"JuliaLang/julia:master\")`")
+build_test_submission("@nanosoldier `runbenchmarks($tagpred; isdaily = true, vs = \"JuliaLang/julia:master\")`")
 
 #########################
 # job report generation #

From a5b136222f1cbfde4d9638b9ad8316271c7a344b Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Mon, 6 Jun 2016 13:23:58 -0400
Subject: [PATCH 03/13] new daily jobs are compared against the results of old
 daily jobs

---
 src/jobs/BenchmarkJob.jl | 70 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 63 insertions(+), 7 deletions(-)

diff --git a/src/jobs/BenchmarkJob.jl b/src/jobs/BenchmarkJob.jl
index 6a26055..b9184df 100644
--- a/src/jobs/BenchmarkJob.jl
+++ b/src/jobs/BenchmarkJob.jl
@@ -81,7 +81,7 @@ function BenchmarkJob(submission::JobSubmission)
     end
 
     return BenchmarkJob(submission, first(submission.args), against,
-                        Dates.today(), skipbuild, isdaily)
+                        Dates.today(), isdaily, skipbuild)
 end
 
 function branchref(config::Config, reponame::AbstractString, branchname::AbstractString)
@@ -108,9 +108,11 @@ end
 
 submission(job::BenchmarkJob) = job.submission
 
+datedirname(date::Dates.Date) = string("daily_", Dates.year(date), "_", Dates.month(date), "_", Dates.day(date))
+
 function jobdirname(job::BenchmarkJob)
     if job.isdaily
-        return string("daily_", year(job.date), "_", month(job.date), "_", day(job.date))
+        return datedirname(job.date)
     else
         primarysha = snipsha(submission(job).build.sha)
         if isnull(job.against)
@@ -133,7 +135,8 @@ function Base.run(job::BenchmarkJob)
     node = myid()
     cfg = submission(job).config
 
-    # update BaseBenchmarks for all Julia versions
+    # update BaseBenchmarks for all supported Julia versions
+    nodelog(cfg, node, "updating local BaseBenchmarks repo")
     branchname = cfg.testmode ? "test" : "nanosoldier"
     oldpwd = pwd()
     versiondirs = ("v0.4", "v0.5")
@@ -145,6 +148,10 @@ function Base.run(job::BenchmarkJob)
     cd(oldpwd)
 
     # make data directory for job
+    nodelog(cfg, node, "creating job directories in report repository")
+    nodelog(cfg, node, "...creating $(reportdir(job))...")
+    mkdir(reportdir(job))
+    nodelog(cfg, node, "...creating $(datadir(job))...")
     mkdir(datadir(job))
 
     # run primary job
@@ -153,12 +160,47 @@ function Base.run(job::BenchmarkJob)
     nodelog(cfg, node, "finished primary build for $(summary(job))")
     results = Dict("primary" => primary_results)
 
-    # run comparison job
-    if !(isnull(job.against))
+    # gather results to compare against
+    if job.isdaily # get results from previous day (if it exists, check the past 30 days)
+        nodelog(cfg, node, "retrieving results from previous daily build")
+        found_previous_date = false
+        i = 1
+        while !(found_previous_date) && i < 31
+            check_date = job.date - Dates.Day(i)
+            if isdir(joinpath(repordir(cfg), datedirname(check_date)))
+                results["previous_date"] = check_date
+                found_previous_date = true
+            end
+            i += 1
+        end
+        if found_previous_date # untar and retrieve old data
+            try
+                previous_path = joinpath(repordir(cfg), datedirname(results["previous_date"]))
+                cd(previous_path) do
+                    run(`tar -xvzf data.tar.gz`)
+                    datapath = joinpath(previous_path, "data")
+                    try
+                        datafiles = readdir(datapath)
+                        jldfile = datafiles[findfirst(fname -> endswith(fname, "_primary.jld"), datafiles)]
+                        results["against"] = JLD.load(joinpath(datapath, jldfile), "results")
+                    catch err
+                        throw(err)
+                    finally
+                        rm(datapath, recursive = true)
+                    end
+                end
+            catch err
+                nodelog(cfg, node, "encountered error when retrieving old daily build data: $(err)")
+            end
+        end
+    elseif !(isnull(job.against)) # run comparison build
         nodelog(cfg, node, "running comparison build for $(summary(job))")
         against_results = execute_benchmarks!(job, :against)
         nodelog(cfg, node, "finished comparison build for $(summary(job))")
         results["against"] = against_results
+    end
+
+    if haskey(results, "against")
         results["judged"] = BenchmarkTools.judge(minimum(primary_results), minimum(against_results))
     end
 
@@ -391,7 +433,23 @@ function printreport(io::IO, job::BenchmarkJob, results)
                 *Triggered By:* [link]($(submission(job).url))
 
                 *Tag Predicate:* `$(job.tagpred)`
+                """)
+
+    if job.isdaily
+        if haskey(results, "previous_date")
+            dailystr = string(job.date, " vs ", results["previous_date"])
+        else
+            dailystr = string(job.date)
+        end
+        println(io, """
+                    *Daily Job:* $(dailystr)
+                    """)
+    end
+
+    # print result table #
+    #--------------------#
 
+    println(io, """
                 ## Results
 
                 *Note: If Chrome is your browser, I strongly recommend installing the [Wide GitHub](https://chrome.google.com/webstore/detail/wide-github/kaalofacklcidaampbokdplbklpeldpj?hl=en)
@@ -407,8 +465,6 @@ function printreport(io::IO, job::BenchmarkJob, results)
                 time/memory value for a given benchmark is expected to fall within this percentage of the reported value.
                 """)
 
-    # print result table #
-    #--------------------#
     if iscomparisonjob
         print(io, """
                   A ratio greater than `1.0` denotes a possible regression (marked with $(REGRESS_MARK)), while a ratio less

From 987230b835b507251e6bd3880ae0de1248e006b7 Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Mon, 6 Jun 2016 22:46:58 -0400
Subject: [PATCH 04/13] give logs their own subdirectory in the job report path

---
 src/build.jl             |  6 +++---
 src/config.jl            |  8 ++------
 src/jobs/BenchmarkJob.jl | 23 ++++++++++++++---------
 3 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/src/build.jl b/src/build.jl
index c29b90d..5c80dd2 100644
--- a/src/build.jl
+++ b/src/build.jl
@@ -19,7 +19,7 @@ end
 Base.summary(build::BuildRef) = string(build.repo, SHA_SEPARATOR, snipsha(build.sha))
 
 # if a PR number is included, attempt to build from the PR's merge commit
-function build_julia!(config::Config, build::BuildRef, prnumber::Nullable{Int} = Nullable{Int}())
+function build_julia!(config::Config, build::BuildRef, logpath, prnumber::Nullable{Int} = Nullable{Int}())
     # make a temporary workdir for our build
     builddir = mktempdir(workdir(config))
     cd(workdir(config))
@@ -47,8 +47,8 @@ function build_julia!(config::Config, build::BuildRef, prnumber::Nullable{Int} =
 
     # set up logs for STDOUT and STDERR
     logname = string(build.sha, "_build")
-    outfile = joinpath(logdir(config), string(logname, ".out"))
-    errfile = joinpath(logdir(config), string(logname, ".err"))
+    outfile = joinpath(logpath, string(logname, ".out"))
+    errfile = joinpath(logpath, string(logname, ".err"))
 
     # run the build
     run(pipeline(`make`, stdout = outfile, stderr = errfile))
diff --git a/src/config.jl b/src/config.jl
index 2c96cac..e856b2f 100644
--- a/src/config.jl
+++ b/src/config.jl
@@ -30,14 +30,10 @@ reportrepo(config::Config) = config.reportrepo
 # the local directory of the report repository
 reportdir(config::Config) = joinpath(workdir(config), split(reportrepo(config), "/")[2])
 
-# the directory where build logs are stored
-logdir(config::Config) = joinpath(workdir(config), "logs")
-
 persistdir!(path) = (!(isdir(path)) && mkdir(path); return path)
 
 function persistdir!(config::Config)
     persistdir!(workdir(config))
-    persistdir!(logdir(config))
     if isdir(reportdir(config))
         gitreset!(reportdir(config))
     else
@@ -46,8 +42,8 @@ function persistdir!(config::Config)
 end
 
 function nodelog(config::Config, node, message)
-    persistdir!(logdir(config))
-    open(joinpath(logdir(config), "node$(node).log"), "a") do file
+    persistdir!(workdir(config))
+    open(joinpath(workdir(config), "node$(node).log"), "a") do file
         println(file, now(), " | ", node, " | ", message)
     end
 end
diff --git a/src/jobs/BenchmarkJob.jl b/src/jobs/BenchmarkJob.jl
index b9184df..0c4d333 100644
--- a/src/jobs/BenchmarkJob.jl
+++ b/src/jobs/BenchmarkJob.jl
@@ -91,7 +91,9 @@ end
 
 function Base.summary(job::BenchmarkJob)
     result = "BenchmarkJob $(summary(submission(job).build))"
-    if !(isnull(job.against))
+    if job.isdaily
+        result = "$(result) [daily]"
+    elseif !(isnull(job.against))
         result = "$(result) vs. $(summary(get(job.against)))"
     end
     return result
@@ -125,6 +127,7 @@ function jobdirname(job::BenchmarkJob)
 end
 
 reportdir(job::BenchmarkJob) = joinpath(reportdir(submission(job).config), jobdirname(job))
+logdir(job::BenchmarkJob) = joinpath(reportdir(job), "logs")
 datadir(job::BenchmarkJob) = joinpath(reportdir(job), "data")
 
 ##########################
@@ -151,6 +154,8 @@ function Base.run(job::BenchmarkJob)
     nodelog(cfg, node, "creating job directories in report repository")
     nodelog(cfg, node, "...creating $(reportdir(job))...")
     mkdir(reportdir(job))
+    nodelog(cfg, node, "...creating $(logdir(job))...")
+    mkdir(logdir(job))
     nodelog(cfg, node, "...creating $(datadir(job))...")
     mkdir(datadir(job))
 
@@ -167,7 +172,7 @@ function Base.run(job::BenchmarkJob)
         i = 1
         while !(found_previous_date) && i < 31
             check_date = job.date - Dates.Day(i)
-            if isdir(joinpath(repordir(cfg), datedirname(check_date)))
+            if isdir(joinpath(reportdir(cfg), datedirname(check_date)))
                 results["previous_date"] = check_date
                 found_previous_date = true
             end
@@ -175,7 +180,7 @@ function Base.run(job::BenchmarkJob)
         end
         if found_previous_date # untar and retrieve old data
             try
-                previous_path = joinpath(repordir(cfg), datedirname(results["previous_date"]))
+                previous_path = joinpath(reportdir(cfg), datedirname(results["previous_date"]))
                 cd(previous_path) do
                     run(`tar -xvzf data.tar.gz`)
                     datapath = joinpath(previous_path, "data")
@@ -224,9 +229,9 @@ function execute_benchmarks!(job::BenchmarkJob, whichbuild::Symbol)
         # If we're doing the primary build from a PR, feed `build_julia!` the PR number
         # so that it knows to attempt a build from the merge commit
         if whichbuild == :primary && submission(job).fromkind == :pr
-            builddir = build_julia!(cfg, build, submission(job).prnumber)
+            builddir = build_julia!(cfg, build, logdir(job), submission(job).prnumber)
         else
-            builddir = build_julia!(cfg, build)
+            builddir = build_julia!(cfg, build, logdir(job))
         end
         juliapath = joinpath(builddir, "julia")
     end
@@ -265,8 +270,8 @@ function execute_benchmarks!(job::BenchmarkJob, whichbuild::Symbol)
     end
 
     benchname = string(build.sha, "_", whichbuild)
-    benchout = joinpath(reportdir(job), string(benchname, ".out"))
-    bencherr = joinpath(reportdir(job), string(benchname, ".err"))
+    benchout = joinpath(logdir(job), string(benchname, ".out"))
+    bencherr = joinpath(logdir(job), string(benchname, ".err"))
     benchresults = joinpath(datadir(job), string(benchname, ".jld"))
 
     open(jlscriptpath, "w") do file
@@ -377,11 +382,11 @@ function report(job::BenchmarkJob, results)
             # write the markdown report
             nodelog(cfg, node, "...generating report...")
             reportname = "report.md"
-            open(joinpath(reportdir(job), reportname)) do file
+            open(joinpath(reportdir(job), reportname), "w") do file
                 printreport(file, job, results)
             end
             # push changes to report repo
-            target_url = upload_report_repo!(job, joinpath(jobdirname(job), reportname), "upload markdown report for $(summary(job))")
+            target_url = upload_report_repo!(job, joinpath(jobdirname(job), reportname), "upload report for $(summary(job))")
         catch err
             nodelog(cfg, node, "error when pushing to report repo: $(err)")
         end

From 00dadac70a3e1cb507e67e84977e9f828b3f3b12 Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Tue, 7 Jun 2016 10:02:53 -0400
Subject: [PATCH 05/13] use the minimum estimator when generating report if the
 data is a Trial

---
 src/jobs/BenchmarkJob.jl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/jobs/BenchmarkJob.jl b/src/jobs/BenchmarkJob.jl
index 0c4d333..78232e5 100644
--- a/src/jobs/BenchmarkJob.jl
+++ b/src/jobs/BenchmarkJob.jl
@@ -543,6 +543,8 @@ idrepr(id) = (str = repr(id); str[searchindex(str, '['):end])
 
 intpercent(p) = string(ceil(Int, p * 100), "%")
 
+resultrow(ids, t::BenchmarkTools.Trial) = resultrow(ids, minimum(t))
+
 function resultrow(ids, t::BenchmarkTools.TrialEstimate)
     t_tol = intpercent(BenchmarkTools.params(t).time_tolerance)
     m_tol = intpercent(BenchmarkTools.params(t).memory_tolerance)

From b0f802e18c3428660aa34760b2d83f516176b79c Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Tue, 7 Jun 2016 11:49:43 -0400
Subject: [PATCH 06/13] remove old job report directories if they exist

---
 src/jobs/BenchmarkJob.jl | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/jobs/BenchmarkJob.jl b/src/jobs/BenchmarkJob.jl
index 78232e5..c0a0e66 100644
--- a/src/jobs/BenchmarkJob.jl
+++ b/src/jobs/BenchmarkJob.jl
@@ -151,6 +151,10 @@ function Base.run(job::BenchmarkJob)
     cd(oldpwd)
 
     # make data directory for job
+    if isdir(reportdir(job))
+        nodelog(cfg, node, "removing old job directory from report repository")
+        rm(reportdir(job), recursive = true)
+    end
     nodelog(cfg, node, "creating job directories in report repository")
     nodelog(cfg, node, "...creating $(reportdir(job))...")
     mkdir(reportdir(job))

From c1334fefe32cef379b930e0afeaafe17d09b8e43 Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Tue, 7 Jun 2016 12:07:40 -0400
Subject: [PATCH 07/13] use SSH instead of HTTPS for repo cloning, do not clone
 report repo onto master node

---
 src/Nanosoldier.jl | 2 +-
 src/server.jl      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Nanosoldier.jl b/src/Nanosoldier.jl
index dbc5b03..8c94769 100644
--- a/src/Nanosoldier.jl
+++ b/src/Nanosoldier.jl
@@ -11,7 +11,7 @@ const BRANCH_SEPARATOR = ':'
 snip(str, len) = length(str) > len ? str[1:len] : str
 snipsha(sha) = snip(sha, 7)
 
-gitclone!(repo, path) = run(`git clone https://github.com/$(repo) $(path)`)
+gitclone!(repo, path) = run(`git clone git@github.com:$(repo).git $(path)`)
 
 gitreset!() = run(`git fetch --all && git reset --hard origin/master`)
 gitreset!(path) = cd(gitreset!, path)
diff --git a/src/server.jl b/src/server.jl
index 4c20ec0..5a1bc4e 100644
--- a/src/server.jl
+++ b/src/server.jl
@@ -45,7 +45,7 @@ end
 
 function Base.run(server::Server, args...; kwargs...)
     @assert myid() == 1 "Nanosoldier server must be run from the master node"
-    persistdir!(server.config)
+    persistdir!(workdir(server.config))
     # Schedule a task for each node that feeds the node a job from the
     # queque once the node has completed its primary job. If the queue is
     # empty, then the task will call `yield` in order to avoid a deadlock.

From e675b15f014b323e7afdd5c0a7b11a4df60f4351 Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Tue, 7 Jun 2016 14:33:47 -0400
Subject: [PATCH 08/13] separate fetch and reset commands

---
 src/Nanosoldier.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Nanosoldier.jl b/src/Nanosoldier.jl
index 8c94769..34afa73 100644
--- a/src/Nanosoldier.jl
+++ b/src/Nanosoldier.jl
@@ -13,7 +13,7 @@ snipsha(sha) = snip(sha, 7)
 
 gitclone!(repo, path) = run(`git clone git@github.com:$(repo).git $(path)`)
 
-gitreset!() = run(`git fetch --all && git reset --hard origin/master`)
+gitreset!() = (run(`git fetch --all`); run(`git reset --hard origin/master`))
 gitreset!(path) = cd(gitreset!, path)
 
 include("config.jl")

From 3f4cdc41d1912645cb5f1736c6455c33f055816f Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Tue, 7 Jun 2016 16:12:41 -0400
Subject: [PATCH 09/13] fix incorrect report directory naming bug

---
 src/jobs/BenchmarkJob.jl | 50 +++++++++++++++++++++++-----------------
 1 file changed, 29 insertions(+), 21 deletions(-)

diff --git a/src/jobs/BenchmarkJob.jl b/src/jobs/BenchmarkJob.jl
index c0a0e66..4a23efa 100644
--- a/src/jobs/BenchmarkJob.jl
+++ b/src/jobs/BenchmarkJob.jl
@@ -127,8 +127,9 @@ function jobdirname(job::BenchmarkJob)
 end
 
 reportdir(job::BenchmarkJob) = joinpath(reportdir(submission(job).config), jobdirname(job))
-logdir(job::BenchmarkJob) = joinpath(reportdir(job), "logs")
-datadir(job::BenchmarkJob) = joinpath(reportdir(job), "data")
+tmpdir(job::BenchmarkJob) = joinpath(workdir(submission(job).config), "tmpresults")
+tmplogdir(job::BenchmarkJob) = joinpath(tmpdir(job), "logs")
+tmpdatadir(job::BenchmarkJob) = joinpath(tmpdir(job), "data")
 
 ##########################
 # BenchmarkJob Execution #
@@ -150,18 +151,23 @@ function Base.run(job::BenchmarkJob)
     end
     cd(oldpwd)
 
-    # make data directory for job
-    if isdir(reportdir(job))
-        nodelog(cfg, node, "removing old job directory from report repository")
-        rm(reportdir(job), recursive = true)
+    # make temporary directory for job results
+    # Why not create the job's actual report directory now instead? The answer is that
+    # the commit SHA that currently describes the job might change if we find out that
+    # we should use a merge commit instead. To avoid confusion, we dump all the results
+    # to this temporary directory first, then move the data to the correct location
+    # in the reporting phase.
+    nodelog(cfg, node, "creating temporary directory for benchmark results")
+    if isdir(tmpdir(job))
+        nodelog(cfg, node, "...removing old temporary directory...")
+        rm(tmpdir(job), recursive = true)
     end
-    nodelog(cfg, node, "creating job directories in report repository")
-    nodelog(cfg, node, "...creating $(reportdir(job))...")
-    mkdir(reportdir(job))
-    nodelog(cfg, node, "...creating $(logdir(job))...")
-    mkdir(logdir(job))
-    nodelog(cfg, node, "...creating $(datadir(job))...")
-    mkdir(datadir(job))
+    nodelog(cfg, node, "...creating $(tmpdir(job))...")
+    mkdir(tmpdir(job))
+    nodelog(cfg, node, "...creating $(tmplogdir(job))...")
+    mkdir(tmplogdir(job))
+    nodelog(cfg, node, "...creating $(tmpdatadir(job))...")
+    mkdir(tmpdatadir(job))
 
     # run primary job
     nodelog(cfg, node, "running primary build for $(summary(job))")
@@ -233,9 +239,9 @@ function execute_benchmarks!(job::BenchmarkJob, whichbuild::Symbol)
         # If we're doing the primary build from a PR, feed `build_julia!` the PR number
         # so that it knows to attempt a build from the merge commit
         if whichbuild == :primary && submission(job).fromkind == :pr
-            builddir = build_julia!(cfg, build, logdir(job), submission(job).prnumber)
+            builddir = build_julia!(cfg, build, tmplogdir(job), submission(job).prnumber)
         else
-            builddir = build_julia!(cfg, build, logdir(job))
+            builddir = build_julia!(cfg, build, tmplogdir(job))
         end
         juliapath = joinpath(builddir, "julia")
     end
@@ -274,9 +280,9 @@ function execute_benchmarks!(job::BenchmarkJob, whichbuild::Symbol)
     end
 
     benchname = string(build.sha, "_", whichbuild)
-    benchout = joinpath(logdir(job), string(benchname, ".out"))
-    bencherr = joinpath(logdir(job), string(benchname, ".err"))
-    benchresults = joinpath(datadir(job), string(benchname, ".jld"))
+    benchout = joinpath(tmplogdir(job), string(benchname, ".out"))
+    bencherr = joinpath(tmplogdir(job), string(benchname, ".err"))
+    benchresults = joinpath(tmpdatadir(job), string(benchname, ".jld"))
 
     open(jlscriptpath, "w") do file
         println(file, """
@@ -379,16 +385,18 @@ function report(job::BenchmarkJob, results)
         try
             # prepare the benchmark data for uploading
             nodelog(cfg, node, "...preparing data...")
-            cd(reportdir(job)) do
+            cd(tmpdir(job)) do
                 run(`tar -zcvf data.tar.gz data`)
-                rm(datadir(job), recursive = true)
+                rm(tmpdatadir(job), recursive = true)
             end
             # write the markdown report
             nodelog(cfg, node, "...generating report...")
             reportname = "report.md"
-            open(joinpath(reportdir(job), reportname), "w") do file
+            open(joinpath(tmpdir(job), reportname), "w") do file
                 printreport(file, job, results)
             end
+            nodelog(cfg, node, "...moving $(tmpdir(job)) to $(reportdir(job))...")
+            mv(tmpdir(job), reportdir(job); remove_destination = true)
             # push changes to report repo
             target_url = upload_report_repo!(job, joinpath(jobdirname(job), reportname), "upload report for $(summary(job))")
         catch err

From 01445d2a7e5f707bab22d06d724f9aa07c15e15e Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Wed, 8 Jun 2016 12:34:16 -0400
Subject: [PATCH 10/13] fix convert error

---
 src/jobs/BenchmarkJob.jl | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/jobs/BenchmarkJob.jl b/src/jobs/BenchmarkJob.jl
index 4a23efa..5c0c0ca 100644
--- a/src/jobs/BenchmarkJob.jl
+++ b/src/jobs/BenchmarkJob.jl
@@ -173,7 +173,7 @@ function Base.run(job::BenchmarkJob)
     nodelog(cfg, node, "running primary build for $(summary(job))")
     primary_results = execute_benchmarks!(job, :primary)
     nodelog(cfg, node, "finished primary build for $(summary(job))")
-    results = Dict("primary" => primary_results)
+    results = Dict{Any,Any}("primary" => primary_results)
 
     # gather results to compare against
     if job.isdaily # get results from previous day (if it exists, check the past 30 days)
@@ -210,13 +210,12 @@ function Base.run(job::BenchmarkJob)
         end
     elseif !(isnull(job.against)) # run comparison build
         nodelog(cfg, node, "running comparison build for $(summary(job))")
-        against_results = execute_benchmarks!(job, :against)
+        results["against"] = execute_benchmarks!(job, :against)
         nodelog(cfg, node, "finished comparison build for $(summary(job))")
-        results["against"] = against_results
     end
 
     if haskey(results, "against")
-        results["judged"] = BenchmarkTools.judge(minimum(primary_results), minimum(against_results))
+        results["judged"] = BenchmarkTools.judge(minimum(primary_results), minimum(results["against"]))
     end
 
     # report results

From 7475254861632c2bd21fbf5e1f7d5ab8bbf0c95f Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Thu, 9 Jun 2016 12:18:59 -0400
Subject: [PATCH 11/13] fix report generation logic for daily comparison builds

---
 src/jobs/BenchmarkJob.jl | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/jobs/BenchmarkJob.jl b/src/jobs/BenchmarkJob.jl
index 5c0c0ca..b04d13b 100644
--- a/src/jobs/BenchmarkJob.jl
+++ b/src/jobs/BenchmarkJob.jl
@@ -370,7 +370,7 @@ function report(job::BenchmarkJob, results)
         reply_comment(job, "[Your benchmark job]($(submission(job).url)) has completed, but no benchmarks were actually executed. Perhaps your tag predicate contains mispelled tags? cc @jrevels")
     else
         # determine the job's final status
-        if !(isnull(job.against))
+        if !(isnull(job.against)) || haskey(results, "previous_date")
             found_regressions = BenchmarkTools.isregression(results["judged"])
             state = found_regressions ? "failure" : "success"
             status = found_regressions ? "possible performance regressions were detected" : "no performance regressions were detected"
@@ -424,13 +424,18 @@ function printreport(io::IO, job::BenchmarkJob, results)
     buildname = string(build.repo, SHA_SEPARATOR, build.sha)
     buildlink = "https://github.com/$(build.repo)/commit/$(build.sha)"
     joblink = "[$(buildname)]($(buildlink))"
-    iscomparisonjob = !(isnull(job.against))
+    hasagainstbuild = !(isnull(job.against))
+    hasprevdate = haskey(results, "previous_date")
+    iscomparisonjob = hasagainstbuild || hasprevdate
 
-    if iscomparisonjob
+    if hasagainstbuild
         againstbuild = get(job.against)
         againstname = string(againstbuild.repo, SHA_SEPARATOR, againstbuild.sha)
         againstlink = "https://github.com/$(againstbuild.repo)/commit/$(againstbuild.sha)"
         joblink = "$(joblink) vs [$(againstname)]($(againstlink))"
+    end
+
+    if iscomparisonjob
         tablegroup = results["judged"]
     else
         tablegroup = results["primary"]
@@ -452,7 +457,7 @@ function printreport(io::IO, job::BenchmarkJob, results)
                 """)
 
     if job.isdaily
-        if haskey(results, "previous_date")
+        if hasprevdate
             dailystr = string(job.date, " vs ", results["previous_date"])
         else
             dailystr = string(job.date)
@@ -538,7 +543,7 @@ function printreport(io::IO, job::BenchmarkJob, results)
               ```
               """)
 
-    if iscomparisonjob
+    if hasagainstbuild
         println(io)
         print(io, """
                   #### Comparison Build

From 1ca553fa0109029bece2946c60150f61865b9545 Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Thu, 9 Jun 2016 14:36:52 -0400
Subject: [PATCH 12/13] explain new result directory structure in README

---
 README.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/README.md b/README.md
index 85b33d1..25f3dd1 100644
--- a/README.md
+++ b/README.md
@@ -60,6 +60,18 @@ The `vs` keyword argument takes a reference string which can points to a Julia c
 - `"owner/repo:branch"`: the head commit of the branch named `branch` in the repository `owner/repo`
 - `"owner/repo@sha"`: the commit specified by `sha` in the repository `owner/repo`
 
+##### Benchmark Results
+
+Once a `BenchmarjJob` is complete, the results are uploaded to the
+[BaseBenchmarkReports](https://github.com/JuliaCI/BaseBenchmarkReports) repository. Each job
+has its own directory for results. This directory contains the following items:
+
+- `report.md` is a markdown report that summarizes the job results
+- `data.tar.gz` contains raw timing data in JLD format. To untar this file, run
+`tar -xzvf data.tar.gz`. You can analyze this data using the
+[BenchmarkTools](https://github.com/JuliaCI/BaseBenchmarkReports) package.
+- `logs` is a directory containing the build logs and benchmark execution logs for the job.
+
 ##### Comment Examples
 
 Here are some examples of comments that trigger a `BenchmarkJob` in various contexts:

From d3ec5cca210d2ca33970b120ff2420867bbcb9ac Mon Sep 17 00:00:00 2001
From: Jarrett Revels <jarrettrevels@gmail.com>
Date: Thu, 9 Jun 2016 14:37:10 -0400
Subject: [PATCH 13/13] add daily job submission script

---
 bin/submit_daily_job.jl | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 bin/submit_daily_job.jl

diff --git a/bin/submit_daily_job.jl b/bin/submit_daily_job.jl
new file mode 100644
index 0000000..8ebd6f4
--- /dev/null
+++ b/bin/submit_daily_job.jl
@@ -0,0 +1,12 @@
+import GitHub
+
+auth = GitHub.authenticate(ENV["GITHUB_AUTH"])
+repo = "JuliaLang/julia"
+sha = get(get(GitHub.branch(repo, "master").commit).sha)
+message = """
+          Executing the daily benchmark build, I will reply here when finished:
+
+          @nanosoldier `runbenchmarks(ALL, isdaily = true)`
+          """
+
+GitHub.create_comment(repo, sha, :commit, auth = auth, params = Dict("body" => message))