diff --git a/src/04_results.jl b/src/04_results.jl index e9d1abc..a0a5f85 100644 --- a/src/04_results.jl +++ b/src/04_results.jl @@ -1,5 +1,5 @@ -# A `Results` object stores information about the results of benchmarking an -# expression. +# A `RawResults` object stores information gained via benchmark execution, +# before any statistical analysis has occurred. # # Fields: # @@ -31,7 +31,7 @@ # time_used::Float64: The time (in nanoseconds) that was consumed by the # benchmarking process. -immutable Results +immutable RawResults precompiled::Bool multiple_samples::Bool search_performed::Bool @@ -40,8 +40,8 @@ immutable Results end # A `SummaryStatistics` object stores the results of a statistic analysis of -# a `Results` object. The precise analysis strategy employed depends on the -# structure of the `Results` object: +# a `RawResults` object. The precise analysis strategy employed depends on the +# structure of the `RawResults` object: # # (1) If only a single sample of a single evaluation was recorded, the # analysis reports only point estimates. @@ -82,7 +82,7 @@ immutable SummaryStatistics allocations::Int r²::Nullable{Float64} - function SummaryStatistics(r::Results) + function SummaryStatistics(r::RawResults) s = r.samples n = length(s.evaluations) n_evaluations = convert(Int, sum(s.evaluations)) @@ -173,137 +173,3 @@ immutable SummaryStatistics ) end end - -function pretty_time_string(t) - if t < 1_000.0 - @sprintf("%.2f ns", t) - elseif t < 1_000_000.0 - @sprintf("%.2f μs", t / 1_000.0) - elseif t < 1_000_000_000.0 - @sprintf("%.2f ms", t / 1_000_000.0) - else # if t < 1_000_000_000_000.0 - @sprintf("%.2f s", t / 1_000_000_000.0) - end -end - -function pretty_memory_string(b) - if b < 1_024.0 - @sprintf("%.2f bytes", b) - elseif b < 1_024.0^2 - @sprintf("%.2f kb", b / 1_024.0) - elseif b < 1_024.0^3 - @sprintf("%.2f mb", b / 1_024.0^2) - else # if b < 1_024.0^4 - @sprintf("%.2f gb", b / 1_024.0^3) - end -end - -# Pretty-print information about the results of benchmarking an expression. -# -# Arguments: -# -# io::IO: An `IO` object to be written to. -# -# r::Results: The `Results` object that we want to print to `io`. -# - -function Base.show(io::IO, r::Results) - stats = SummaryStatistics(r) - - max_length = 24 - @printf(io, "================ Benchmark Results ========================\n") - - if !r.precompiled - @printf(io, "Warning: function may not have been precompiled\n") - end - if isnull(stats.elapsed_time_lower) || isnull(stats.elapsed_time_upper) - @printf( - io, - "%s: %s\n", - lpad("Time per evaluation", max_length), - pretty_time_string(stats.elapsed_time_center), - ) - else - @printf( - io, - "%s: %s [%s, %s]\n", - lpad("Time per evaluation", max_length), - pretty_time_string(stats.elapsed_time_center), - pretty_time_string(get(stats.elapsed_time_lower)), - pretty_time_string(get(stats.elapsed_time_upper)), - ) - end - if isnull(stats.elapsed_time_lower) || isnull(stats.elapsed_time_upper) - @printf( - io, - "%s: %.2f%%\n", - lpad("Proportion of time in GC", max_length), - stats.gc_proportion_center - ) - else - @printf( - io, - "%s: %.2f%% [%.2f%%, %.2f%%]\n", - lpad("Proportion of time in GC", max_length), - stats.gc_proportion_center, - get(stats.gc_proportion_lower), - get(stats.gc_proportion_upper), - ) - end - @printf( - io, - "%s: %s\n", - lpad("Memory allocated", max_length), - pretty_memory_string(stats.bytes_allocated), - ) - @printf( - io, - "%s: %d allocations\n", - lpad("Number of allocations", max_length), - stats.allocations, - ) - @printf( - io, - "%s: %d\n", - lpad("Number of samples", max_length), - stats.n - ) - @printf( - io, - "%s: %d\n", - lpad("Number of evaluations", max_length), - stats.n_evaluations - ) - if r.search_performed - @printf( - io, - "%s: %.3f\n", - lpad("R² of OLS model", max_length), - get(stats.r², NaN), - ) - end - @printf( - io, - "%s: %.2f s\n", - lpad("Time spent benchmarking", max_length), - r.time_used, - ) - # @printf( - # io, - # "%s: %s\n", - # lpad("Precompiled", max_length), - # string(r.precompiled) - # ) - # @printf( - # io, - # "%s: %s\n", - # lpad("Multiple samples", max_length), - # string(r.multiple_samples), - # ) - # @printf( - # io, - # "%s: %s", - # lpad("Search performed", max_length), - # string(r.search_performed), - # ) -end diff --git a/src/05_api.jl b/src/05_api.jl new file mode 100644 index 0000000..696d2ee --- /dev/null +++ b/src/05_api.jl @@ -0,0 +1,168 @@ +# A `BenchmarkResults` object stores both the `RawResults` from benchmark +# execution and the `SummaryStatistics` computed from these `RawResults`. The +# purpose of the `BenchmarkResults` type is to serve as a vessel for the API. +# +# Fields: +# +# raw::RawResults: The raw data gained via benchmark execution +# +# stats::SummaryStatistics: The results of performing statistical analysis +# on the `raw` field + +immutable BenchmarkResults + raw::RawResults + stats::SummaryStatistics + BenchmarkResults(raw::RawResults) = new(raw, SummaryStatistics(raw)) +end + +# Returns the total time (ns) spent executing the benchmark +totaltime(r::BenchmarkResults) = r.raw.time_used + +# Returns estimates of the time (ns) per evaluation for the benchmarked function +timepereval(r::BenchmarkResults) = r.stats.elapsed_time_center +timepereval_lower(r::BenchmarkResults) = r.stats.elapsed_time_lower +timepereval_upper(r::BenchmarkResults) = r.stats.elapsed_time_upper + +# Returns estimates of the % of time spent in GC during benchmark execution +gcpercent(r::BenchmarkResults) = r.stats.gc_proportion_center +gcpercent_lower(r::BenchmarkResults) = r.stats.gc_proportion_lower +gcpercent_upper(r::BenchmarkResults) = r.stats.gc_proportion_upper + +# Returns the # of bytes allocated during benchmark execution +nbytes(r::BenchmarkResults) = r.stats.bytes_allocated + +# Returns the # of allocations made during benchmark execution +nallocs(r::BenchmarkResults) = r.stats.allocations + +# Returns the # of evaluations performed during benchmark execution +nevals(r::BenchmarkResults) = r.stats.n_evaluations + +# Returns the # of samples taken during benchmark execution +nsamples(r::BenchmarkResults) = r.stats.n + +# Returns the r² value of the OLS regression performed on the benchark results +rsquared(r::BenchmarkResults) = r.stats.r² + +# BenchmarkResults pretty-printing functions +function pretty_time_string(t) + if t < 1_000.0 + @sprintf("%.2f ns", t) + elseif t < 1_000_000.0 + @sprintf("%.2f μs", t / 1_000.0) + elseif t < 1_000_000_000.0 + @sprintf("%.2f ms", t / 1_000_000.0) + else # if t < 1_000_000_000_000.0 + @sprintf("%.2f s", t / 1_000_000_000.0) + end +end + +function pretty_memory_string(b) + if b < 1_024.0 + @sprintf("%.2f bytes", b) + elseif b < 1_024.0^2 + @sprintf("%.2f kb", b / 1_024.0) + elseif b < 1_024.0^3 + @sprintf("%.2f mb", b / 1_024.0^2) + else # if b < 1_024.0^4 + @sprintf("%.2f gb", b / 1_024.0^3) + end +end + +function Base.show(io::IO, r::BenchmarkResults) + max_length = 24 + @printf(io, "================ Benchmark Results ========================\n") + + if !r.raw.precompiled + @printf(io, "Warning: function may not have been precompiled\n") + end + if isnull(timepereval_lower(r)) || isnull(timepereval_upper(r)) + @printf( + io, + "%s: %s\n", + lpad("Time per evaluation", max_length), + pretty_time_string(timepereval(r)), + ) + else + @printf( + io, + "%s: %s [%s, %s]\n", + lpad("Time per evaluation", max_length), + pretty_time_string(timepereval(r)), + pretty_time_string(get(timepereval_lower(r))), + pretty_time_string(get(timepereval_upper(r))), + ) + end + if isnull(gcpercent_lower(r)) || isnull(gcpercent_upper(r)) + @printf( + io, + "%s: %.2f%%\n", + lpad("Proportion of time in GC", max_length), + gcpercent(r) + ) + else + @printf( + io, + "%s: %.2f%% [%.2f%%, %.2f%%]\n", + lpad("Proportion of time in GC", max_length), + gcpercent(r), + get(gcpercent_lower(r)), + get(gcpercent_upper(r)), + ) + end + @printf( + io, + "%s: %s\n", + lpad("Memory allocated", max_length), + pretty_memory_string(nbytes(r)), + ) + @printf( + io, + "%s: %d allocations\n", + lpad("Number of allocations", max_length), + nallocs(r), + ) + @printf( + io, + "%s: %d\n", + lpad("Number of samples", max_length), + nsamples(r) + ) + @printf( + io, + "%s: %d\n", + lpad("Number of evaluations", max_length), + nevals(r) + ) + if r.raw.search_performed + @printf( + io, + "%s: %.3f\n", + lpad("R² of OLS model", max_length), + get(rsquared(r), NaN), + ) + end + @printf( + io, + "%s: %.2f s\n", + lpad("Time spent benchmarking", max_length), + totaltime(r), + ) + # @printf( + # io, + # "%s: %s\n", + # lpad("Precompiled", max_length), + # string(r.raw.precompiled) + # ) + # @printf( + # io, + # "%s: %s\n", + # lpad("Multiple samples", max_length), + # string(r.raw.multiple_samples), + # ) + # @printf( + # io, + # "%s: %s", + # lpad("Search performed", max_length), + # string(r.raw.search_performed), + # ) +end diff --git a/src/Benchmarks.jl b/src/Benchmarks.jl index 415be77..21786d0 100644 --- a/src/Benchmarks.jl +++ b/src/Benchmarks.jl @@ -1,10 +1,24 @@ module Benchmarks - export @benchmark + export @benchmark, + BenchmarkResults, + totaltime, + timepereval, + timepereval_lower, + timepereval_upper, + gcpercent, + gcpercent_lower, + gcpercent_upper, + nbytes, + nallocs, + nevals, + nsamples, + rsquared include("01_clock_resolution.jl") include("02_environment.jl") include("03_samples.jl") include("04_results.jl") + include("05_api.jl") include("benchmarkable.jl") include("ols.jl") include("execute.jl") diff --git a/src/benchmark.jl b/src/benchmark.jl index 44dc772..ea398c9 100644 --- a/src/benchmark.jl +++ b/src/benchmark.jl @@ -25,7 +25,7 @@ macro benchmark(core) $(esc(core)), nothing ) - Benchmarks.execute($name) + Benchmarks.BenchmarkResults(Benchmarks.execute($name)) end end end diff --git a/src/benchmarkable.jl b/src/benchmarkable.jl index 62977b9..8d0587a 100644 --- a/src/benchmarkable.jl +++ b/src/benchmarkable.jl @@ -84,7 +84,7 @@ macro benchmarkable(name, setup, core, teardown) # Execute the teardown expression exactly once $(esc(teardown)) - # The caller receives all data via the mutated Results object. + # The caller receives all data via the mutated RawResults object. return end end diff --git a/src/execute.jl b/src/execute.jl index 1fa825f..3bb4eea 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -26,7 +26,7 @@ # # Returns: # -# r::Results: A Results object containing information about the +# r::RawResults: A RawResults object containing information about the # benchmark's full execution history. function execute( @@ -52,7 +52,7 @@ function execute( # We stop benchmarking f! if we've already exhausted our time budget. time_used = time() - start_time if time_used > budget - return Results(false, false, false, s, time_used) + return RawResults(false, false, false, s, time_used) end # We determine the maximum number of samples we could record given @@ -64,7 +64,7 @@ function execute( # We stop benchmarking if running f! one more time would put us over # our time budget. if max_samples < 1 - return Results(false, false, false, s, time_used) + return RawResults(false, false, false, s, time_used) end # Having reached this point, we can afford to record at least one more @@ -92,7 +92,7 @@ function execute( # only requested a single sample. time_used = time() - start_time if time_used > budget || samples == 1 - return Results(true, false, false, s, time_used) + return RawResults(true, false, false, s, time_used) end # Now we determine if the function is so fast that we need to execute the @@ -105,7 +105,7 @@ function execute( max_samples = floor(Integer, remaining_time_ns / debiased_time_ns) n_samples = min(max_samples, samples - 1) f!(s, n_samples, 1) - return Results(true, true, false, s, time() - start_time) + return RawResults(true, true, false, s, time() - start_time) end # If we've reached this far, we are benchmarking a function that is so fast @@ -160,5 +160,5 @@ function execute( n_evals *= α end - return Results(true, true, true, s, time() - start_time) + return RawResults(true, true, true, s, time() - start_time) end diff --git a/test/05_api.jl b/test/05_api.jl new file mode 100644 index 0000000..a26cfbb --- /dev/null +++ b/test/05_api.jl @@ -0,0 +1,19 @@ +module TestAPI + using Benchmarks + using Base.Test + + f(v) = dot(v, rand(length(v))) + results = @benchmark f(rand(10)) + + @test totaltime(results) > 0.0 + @test timepereval(results) > 0.0 + @test timepereval(results) > get(timepereval_lower(results)) > 0.0 + @test get(timepereval_upper(results)) > timepereval(results) + @test gcpercent(results) > 0.0 + @test gcpercent(results) > get(gcpercent_lower(results)) > 0.0 + @test get(gcpercent_upper(results)) > gcpercent(results) + @test nbytes(results) > 0 + @test nallocs(results) > 0 + @test nsamples(results) > 0 + @test get(rsquared(results)) > 0.8 +end diff --git a/test/runtests.jl b/test/runtests.jl index c5ec82d..4797686 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,6 +6,7 @@ my_tests = [ "01_clock_resolution.jl", "02_environment.jl", "03_samples.jl", + "05_api.jl" ] println("Running tests:")