Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Expose a basic API for accessing benchmark results #26

Closed
wants to merge 3 commits into from
Closed

Expose a basic API for accessing benchmark results #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9219707
Results -> RawResults, add BenchmarkResults type on which API methods…
jrevels Oct 1, 2015
f893380
add tests for API
jrevels Oct 1, 2015
87310c5
fix comment phrasing
jrevels Oct 2, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 6 additions & 140 deletions src/04_results.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# A `Results` object stores information about the results of benchmarking an
# expression.
# A `RawResults` object stores information gained via benchmark execution,
# before any statistical analysis has occurred.
#
# Fields:
#
Expand Down Expand Up @@ -31,7 +31,7 @@
# time_used::Float64: The time (in nanoseconds) that was consumed by the
# benchmarking process.

immutable Results
immutable RawResults
precompiled::Bool
multiple_samples::Bool
search_performed::Bool
Expand All @@ -40,8 +40,8 @@ immutable Results
end

# A `SummaryStatistics` object stores the results of a statistic analysis of
# a `Results` object. The precise analysis strategy employed depends on the
# structure of the `Results` object:
# a `RawResults` object. The precise analysis strategy employed depends on the
# structure of the `RawResults` object:
#
# (1) If only a single sample of a single evaluation was recorded, the
# analysis reports only point estimates.
Expand Down Expand Up @@ -82,7 +82,7 @@ immutable SummaryStatistics
allocations::Int
r²::Nullable{Float64}

function SummaryStatistics(r::Results)
function SummaryStatistics(r::RawResults)
s = r.samples
n = length(s.evaluations)
n_evaluations = convert(Int, sum(s.evaluations))
Expand Down Expand Up @@ -173,137 +173,3 @@ immutable SummaryStatistics
)
end
end

function pretty_time_string(t)
if t < 1_000.0
@sprintf("%.2f ns", t)
elseif t < 1_000_000.0
@sprintf("%.2f μs", t / 1_000.0)
elseif t < 1_000_000_000.0
@sprintf("%.2f ms", t / 1_000_000.0)
else # if t < 1_000_000_000_000.0
@sprintf("%.2f s", t / 1_000_000_000.0)
end
end

function pretty_memory_string(b)
if b < 1_024.0
@sprintf("%.2f bytes", b)
elseif b < 1_024.0^2
@sprintf("%.2f kb", b / 1_024.0)
elseif b < 1_024.0^3
@sprintf("%.2f mb", b / 1_024.0^2)
else # if b < 1_024.0^4
@sprintf("%.2f gb", b / 1_024.0^3)
end
end

# Pretty-print information about the results of benchmarking an expression.
#
# Arguments:
#
# io::IO: An `IO` object to be written to.
#
# r::Results: The `Results` object that we want to print to `io`.
#

function Base.show(io::IO, r::Results)
stats = SummaryStatistics(r)

max_length = 24
@printf(io, "================ Benchmark Results ========================\n")

if !r.precompiled
@printf(io, "Warning: function may not have been precompiled\n")
end
if isnull(stats.elapsed_time_lower) || isnull(stats.elapsed_time_upper)
@printf(
io,
"%s: %s\n",
lpad("Time per evaluation", max_length),
pretty_time_string(stats.elapsed_time_center),
)
else
@printf(
io,
"%s: %s [%s, %s]\n",
lpad("Time per evaluation", max_length),
pretty_time_string(stats.elapsed_time_center),
pretty_time_string(get(stats.elapsed_time_lower)),
pretty_time_string(get(stats.elapsed_time_upper)),
)
end
if isnull(stats.elapsed_time_lower) || isnull(stats.elapsed_time_upper)
@printf(
io,
"%s: %.2f%%\n",
lpad("Proportion of time in GC", max_length),
stats.gc_proportion_center
)
else
@printf(
io,
"%s: %.2f%% [%.2f%%, %.2f%%]\n",
lpad("Proportion of time in GC", max_length),
stats.gc_proportion_center,
get(stats.gc_proportion_lower),
get(stats.gc_proportion_upper),
)
end
@printf(
io,
"%s: %s\n",
lpad("Memory allocated", max_length),
pretty_memory_string(stats.bytes_allocated),
)
@printf(
io,
"%s: %d allocations\n",
lpad("Number of allocations", max_length),
stats.allocations,
)
@printf(
io,
"%s: %d\n",
lpad("Number of samples", max_length),
stats.n
)
@printf(
io,
"%s: %d\n",
lpad("Number of evaluations", max_length),
stats.n_evaluations
)
if r.search_performed
@printf(
io,
"%s: %.3f\n",
lpad("R² of OLS model", max_length),
get(stats.r², NaN),
)
end
@printf(
io,
"%s: %.2f s\n",
lpad("Time spent benchmarking", max_length),
r.time_used,
)
# @printf(
# io,
# "%s: %s\n",
# lpad("Precompiled", max_length),
# string(r.precompiled)
# )
# @printf(
# io,
# "%s: %s\n",
# lpad("Multiple samples", max_length),
# string(r.multiple_samples),
# )
# @printf(
# io,
# "%s: %s",
# lpad("Search performed", max_length),
# string(r.search_performed),
# )
end
168 changes: 168 additions & 0 deletions src/05_api.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
# A `BenchmarkResults` object stores both the `RawResults` from benchmark
# execution and the `SummaryStatistics` computed from these `RawResults`. The
# purpose of the `BenchmarkResults` type is to serve as a vessel for the API.
#
# Fields:
#
# raw::RawResults: The raw data gained via benchmark execution
#
# stats::SummaryStatistics: The results of performing statistical analysis
# on the `raw` field

immutable BenchmarkResults
raw::RawResults
stats::SummaryStatistics
BenchmarkResults(raw::RawResults) = new(raw, SummaryStatistics(raw))
end

# Returns the total time (ns) spent executing the benchmark
totaltime(r::BenchmarkResults) = r.raw.time_used

# Returns estimates of the time (ns) per evaluation for the benchmarked function
timepereval(r::BenchmarkResults) = r.stats.elapsed_time_center
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe return tuples instead of having separate functions?

timepereval_lower(r::BenchmarkResults) = r.stats.elapsed_time_lower
timepereval_upper(r::BenchmarkResults) = r.stats.elapsed_time_upper

# Returns estimates of the % of time spent in GC during benchmark execution
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same tuple idea would work here.

gcpercent(r::BenchmarkResults) = r.stats.gc_proportion_center
gcpercent_lower(r::BenchmarkResults) = r.stats.gc_proportion_lower
gcpercent_upper(r::BenchmarkResults) = r.stats.gc_proportion_upper

# Returns the # of bytes allocated during benchmark execution
nbytes(r::BenchmarkResults) = r.stats.bytes_allocated

# Returns the # of allocations made during benchmark execution
nallocs(r::BenchmarkResults) = r.stats.allocations

# Returns the # of evaluations performed during benchmark execution
nevals(r::BenchmarkResults) = r.stats.n_evaluations

# Returns the # of samples taken during benchmark execution
nsamples(r::BenchmarkResults) = r.stats.n

# Returns the r² value of the OLS regression performed on the benchark results
rsquared(r::BenchmarkResults) = r.stats.r²

# BenchmarkResults pretty-printing functions
function pretty_time_string(t)
if t < 1_000.0
@sprintf("%.2f ns", t)
elseif t < 1_000_000.0
@sprintf("%.2f μs", t / 1_000.0)
elseif t < 1_000_000_000.0
@sprintf("%.2f ms", t / 1_000_000.0)
else # if t < 1_000_000_000_000.0
@sprintf("%.2f s", t / 1_000_000_000.0)
end
end

function pretty_memory_string(b)
if b < 1_024.0
@sprintf("%.2f bytes", b)
elseif b < 1_024.0^2
@sprintf("%.2f kb", b / 1_024.0)
elseif b < 1_024.0^3
@sprintf("%.2f mb", b / 1_024.0^2)
else # if b < 1_024.0^4
@sprintf("%.2f gb", b / 1_024.0^3)
end
end

function Base.show(io::IO, r::BenchmarkResults)
max_length = 24
@printf(io, "================ Benchmark Results ========================\n")

if !r.raw.precompiled
@printf(io, "Warning: function may not have been precompiled\n")
end
if isnull(timepereval_lower(r)) || isnull(timepereval_upper(r))
@printf(
io,
"%s: %s\n",
lpad("Time per evaluation", max_length),
pretty_time_string(timepereval(r)),
)
else
@printf(
io,
"%s: %s [%s, %s]\n",
lpad("Time per evaluation", max_length),
pretty_time_string(timepereval(r)),
pretty_time_string(get(timepereval_lower(r))),
pretty_time_string(get(timepereval_upper(r))),
)
end
if isnull(gcpercent_lower(r)) || isnull(gcpercent_upper(r))
@printf(
io,
"%s: %.2f%%\n",
lpad("Proportion of time in GC", max_length),
gcpercent(r)
)
else
@printf(
io,
"%s: %.2f%% [%.2f%%, %.2f%%]\n",
lpad("Proportion of time in GC", max_length),
gcpercent(r),
get(gcpercent_lower(r)),
get(gcpercent_upper(r)),
)
end
@printf(
io,
"%s: %s\n",
lpad("Memory allocated", max_length),
pretty_memory_string(nbytes(r)),
)
@printf(
io,
"%s: %d allocations\n",
lpad("Number of allocations", max_length),
nallocs(r),
)
@printf(
io,
"%s: %d\n",
lpad("Number of samples", max_length),
nsamples(r)
)
@printf(
io,
"%s: %d\n",
lpad("Number of evaluations", max_length),
nevals(r)
)
if r.raw.search_performed
@printf(
io,
"%s: %.3f\n",
lpad("R² of OLS model", max_length),
get(rsquared(r), NaN),
)
end
@printf(
io,
"%s: %.2f s\n",
lpad("Time spent benchmarking", max_length),
totaltime(r),
)
# @printf(
# io,
# "%s: %s\n",
# lpad("Precompiled", max_length),
# string(r.raw.precompiled)
# )
# @printf(
# io,
# "%s: %s\n",
# lpad("Multiple samples", max_length),
# string(r.raw.multiple_samples),
# )
# @printf(
# io,
# "%s: %s",
# lpad("Search performed", max_length),
# string(r.raw.search_performed),
# )
end
16 changes: 15 additions & 1 deletion src/Benchmarks.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,24 @@
module Benchmarks
export @benchmark
export @benchmark,
BenchmarkResults,
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would prefer not to export so many names. I'm trying to do my part to remove the culture of using pulling in a ton of names.

totaltime,
timepereval,
timepereval_lower,
timepereval_upper,
gcpercent,
gcpercent_lower,
gcpercent_upper,
nbytes,
nallocs,
nevals,
nsamples,
rsquared

include("01_clock_resolution.jl")
include("02_environment.jl")
include("03_samples.jl")
include("04_results.jl")
include("05_api.jl")
include("benchmarkable.jl")
include("ols.jl")
include("execute.jl")
Expand Down
2 changes: 1 addition & 1 deletion src/benchmark.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ macro benchmark(core)
$(esc(core)),
nothing
)
Benchmarks.execute($name)
Benchmarks.BenchmarkResults(Benchmarks.execute($name))
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we should auto-generate summary statistics for all benchmarks. This makes sense now because the statistical analysis is fairly cheap. If we switch to something like bootstrapping by default, it's less clear to me that we should force people to pay the cost for an analysis they won't use.

end
end
end
2 changes: 1 addition & 1 deletion src/benchmarkable.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ macro benchmarkable(name, setup, core, teardown)
# Execute the teardown expression exactly once
$(esc(teardown))

# The caller receives all data via the mutated Results object.
# The caller receives all data via the mutated RawResults object.
return
end
end
Expand Down
Loading