forked from sdd/kiddo_v1
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathgenerate_benchmark_charts.py
139 lines (121 loc) · 5.45 KB
/
generate_benchmark_charts.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
import matplotlib.pyplot as plt
import ndjson
chartgroup_defs = [
{
'charts': [
{ 'source': 'add 100 items to 2d kdtree of increasing size', 'title': '2d tree' },
{ 'source': 'add 100 items to 3d kdtree (f32) of increasing size', 'title': '3d tree (f32)' },
{ 'source': 'add 100 items to 3d kdtree of increasing size', 'title': '3d tree (f64)' },
{ 'source': 'add 100 items to 4d kdtree of increasing size', 'title': '4d tree' },
],
'log_time': True,
'n_rows': 1,
'group_title': "Adding items",
'output_filename': "benchmark_adding"
},
{
'charts': [
{ 'source': 'nearest(1)', 'title': 'nearest 1 item' },
{ 'source': 'nearest(100)', 'title': 'nearest 100 items' },
{ 'source': 'nearest(1000)', 'title': 'nearest 1000 items' },
],
'log_time': True,
'n_rows': 1,
'group_title': "Querying: Nearest n Items (sorted)",
'output_filename': "benchmark_nearest_n"
},
{
'charts': [
{ 'source': 'within(0.01)', 'title': 'items within 0.01 dist2' },
{ 'source': 'within(0.05)', 'title': 'items within 0.05 dist2' },
{ 'source': 'within(0.25)', 'title': 'items within 0.25 dist2' },
],
'log_time': True,
'n_rows': 1,
'group_title': "Querying: all items within specified distance of query point",
'output_filename': "benchmark_within"
},
{
'charts': [
{ 'source': 'within_unsorted(0.01)', 'title': 'unsorted items within 0.01 dist2' },
{ 'source': 'within_unsorted(0.05)', 'title': 'unsorted items within 0.05 dist2' },
{ 'source': 'within_unsorted(0.25)', 'title': 'unsorted items within 0.25 dist2' },
],
'log_time': True,
'n_rows': 1,
'group_title': "Querying: all items (unsorted) within specified distance of query point",
'output_filename': "benchmark_within_unsorted"
},
{
'charts': [
{ 'source': 'best 1: within(0.01)', 'title': 'best 1 item within 0.01 dist2' },
{ 'source': 'best 1: within(0.05)', 'title': 'best 1 item within 0.05 dist2' },
{ 'source': 'best 1: within(0.25)', 'title': 'best 1 item within 0.25 dist2' },
{ 'source': 'best 100: within(0.01)', 'title': 'best 100 items within 0.01 dist2' },
{ 'source': 'best 100: within(0.05)', 'title': 'best 100 items within 0.05 dist2' },
{ 'source': 'best 100: within(0.25)', 'title': 'best 100 items within 0.25 dist2' },
],
'log_time': True,
'n_rows': 2,
'group_title': "Querying: Best n items within specified distance of query point",
'output_filename': "benchmark_best_n_within"
},
]
def transform_criterion_data(bench_data):
results = {}
for result in bench_data:
if result['reason'] == 'benchmark-complete':
key_frags = result['id'].split('/')
if key_frags[0] not in results:
results[key_frags[0]] = {}
results[key_frags[0]][key_frags[1]] = result['typical']
return results
def data_for_benchmark(data, name):
x = [int(x) for x in data[name].keys()]
y = [r['estimate'] / 1000 for r in data[name].values()]
return [ x, y ]
def render_chart_group(data, data_names, chartdef, y_is_logarithmic, n_rows, sup_title, output_filename):
n_cols = len(chartdef) // n_rows
fig, ax = plt.subplots(nrows=n_rows, ncols=n_cols, figsize=(16,4 * n_rows), facecolor='w', edgecolor='k')
for idx, chart in enumerate(chartdef):
if n_rows == 1:
crt = ax[idx]
else:
crt = ax[idx // n_cols][idx % n_cols]
for series_idx, series in enumerate(data):
d = data_for_benchmark(series, chart['source'])
crt.plot(d[0], d[1], data_names[series_idx][1], label=data_names[series_idx][0])
if y_is_logarithmic:
crt.set_yscale('log')
crt.set_xscale('log')
crt.set_title(chart['title'])
crt.set_xlabel('tree size')
crt.set_ylabel('Time / μs')
if y_is_logarithmic == False:
crt.set_ylim([0, None])
crt.legend()
crt.grid(True, axis='x', which="major", ls="-", color='0.65')
crt.grid(True, axis='y', which='major', linestyle='-', color='dimgrey')
crt.grid(True, axis='y', which='minor', linestyle=':', color='0.45')
if sup_title is not None:
fig.suptitle(sup_title, size='x-large')
plt.tight_layout()
plt.savefig(output_filename + '.png', transparent=False, bbox_inches='tight')
def render_chart_group_def(data, data_labels, chart_group_def):
render_chart_group(data, data_labels, chart_group_def['charts'], chart_group_def['log_time'], chart_group_def['n_rows'], chart_group_def['group_title'], chart_group_def['output_filename'])
def render_chart_group_defs(data, data_labels, chart_group_defs):
for chart_group_def in chart_group_defs:
render_chart_group_def(data, data_labels, chart_group_def)
with open("./criterion-kiddo.ndjson") as datafile:
data_kiddo = ndjson.load(datafile)
with open("./criterion-kdtree.ndjson") as datafile:
data_kdtree = ndjson.load(datafile)
data_labels = [
['kiddo', 'green'],
['kdtree', 'red']
]
data = [
transform_criterion_data(data_kiddo),
transform_criterion_data(data_kdtree)
]
render_chart_group_defs(data, data_labels, chartgroup_defs)