Skip to content

Commit 5f31abd

Browse files
LebedevRILebedevRI
authored
Merge 996adfc into 1b44120 
2 parents 1b44120 + 996adfc commit 5f31abd

File tree

5 files changed

+350
-100
lines changed

5 files changed

+350
-100
lines changed

‎docs/tools.md

+9
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,15 @@ The `compare.py` can be used to compare the result of benchmarks.
66

77
**NOTE**: the utility relies on the scipy package which can be installed using [these instructions](https://www.scipy.org/install.html).
88

9+
### Displaying aggregates only
10+
11+
The switch `-a` / `--display_aggregates_only` can be used to control the
12+
displayment of the normal iterations vs the aggregates. When passed, it will
13+
be passthrough to the benchmark binaries to be run, and will be accounted for
14+
in the tool itself; only the aggregates will be displayed, but not normal runs.
15+
It only affects the display, the separate runs will still be used to calculate
16+
the U test.
17+
918
### Modes of operation
1019

1120
There are three modes of operation:

‎tools/compare.py

+39-2
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,17 @@ def create_parser():
3636
parser = ArgumentParser(
3737
description='versatile benchmark output compare tool')
3838

39+
parser.add_argument(
40+
'-a',
41+
'--display_aggregates_only',
42+
dest='display_aggregates_only',
43+
action="store_true",
44+
help="If there are repetitions, by default, we display everything - the"
45+
" actual runs, and the aggregates computed. Sometimes, it is "
46+
"desirable to only view the aggregates. E.g. when there are a lot "
47+
"of repetitions. Do note that only the display is affected. "
48+
"Internally, all the actual runs are still used, e.g. for U test.")
49+
3950
utest = parser.add_argument_group()
4051
utest.add_argument(
4152
'--no-utest',
@@ -200,6 +211,9 @@ def main():
200211

201212
check_inputs(test_baseline, test_contender, benchmark_options)
202213

214+
if args.display_aggregates_only:
215+
benchmark_options += ['--benchmark_display_aggregates_only=true']
216+
203217
options_baseline = []
204218
options_contender = []
205219

@@ -223,7 +237,8 @@ def main():
223237

224238
# Diff and output
225239
output_lines = gbench.report.generate_difference_report(
226-
json1, json2, args.utest, args.utest_alpha)
240+
json1, json2, args.display_aggregates_only,
241+
args.utest, args.utest_alpha)
227242
print(description)
228243
for ln in output_lines:
229244
print(ln)
@@ -246,6 +261,7 @@ def setUp(self):
246261
def test_benchmarks_basic(self):
247262
parsed = self.parser.parse_args(
248263
['benchmarks', self.testInput0, self.testInput1])
264+
self.assertFalse(parsed.display_aggregates_only)
249265
self.assertTrue(parsed.utest)
250266
self.assertEqual(parsed.mode, 'benchmarks')
251267
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -255,16 +271,28 @@ def test_benchmarks_basic(self):
255271
def test_benchmarks_basic_without_utest(self):
256272
parsed = self.parser.parse_args(
257273
['--no-utest', 'benchmarks', self.testInput0, self.testInput1])
274+
self.assertFalse(parsed.display_aggregates_only)
258275
self.assertFalse(parsed.utest)
259276
self.assertEqual(parsed.utest_alpha, 0.05)
260277
self.assertEqual(parsed.mode, 'benchmarks')
261278
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
262279
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
263280
self.assertFalse(parsed.benchmark_options)
264281

282+
def test_benchmarks_basic_display_aggregates_only(self):
283+
parsed = self.parser.parse_args(
284+
['-a', 'benchmarks', self.testInput0, self.testInput1])
285+
self.assertTrue(parsed.display_aggregates_only)
286+
self.assertTrue(parsed.utest)
287+
self.assertEqual(parsed.mode, 'benchmarks')
288+
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
289+
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
290+
self.assertFalse(parsed.benchmark_options)
291+
265292
def test_benchmarks_basic_with_utest_alpha(self):
266293
parsed = self.parser.parse_args(
267294
['--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1])
295+
self.assertFalse(parsed.display_aggregates_only)
268296
self.assertTrue(parsed.utest)
269297
self.assertEqual(parsed.utest_alpha, 0.314)
270298
self.assertEqual(parsed.mode, 'benchmarks')
@@ -275,6 +303,7 @@ def test_benchmarks_basic_with_utest_alpha(self):
275303
def test_benchmarks_basic_without_utest_with_utest_alpha(self):
276304
parsed = self.parser.parse_args(
277305
['--no-utest', '--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1])
306+
self.assertFalse(parsed.display_aggregates_only)
278307
self.assertFalse(parsed.utest)
279308
self.assertEqual(parsed.utest_alpha, 0.314)
280309
self.assertEqual(parsed.mode, 'benchmarks')
@@ -285,6 +314,7 @@ def test_benchmarks_basic_without_utest_with_utest_alpha(self):
285314
def test_benchmarks_with_remainder(self):
286315
parsed = self.parser.parse_args(
287316
['benchmarks', self.testInput0, self.testInput1, 'd'])
317+
self.assertFalse(parsed.display_aggregates_only)
288318
self.assertTrue(parsed.utest)
289319
self.assertEqual(parsed.mode, 'benchmarks')
290320
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -294,6 +324,7 @@ def test_benchmarks_with_remainder(self):
294324
def test_benchmarks_with_remainder_after_doubleminus(self):
295325
parsed = self.parser.parse_args(
296326
['benchmarks', self.testInput0, self.testInput1, '--', 'e'])
327+
self.assertFalse(parsed.display_aggregates_only)
297328
self.assertTrue(parsed.utest)
298329
self.assertEqual(parsed.mode, 'benchmarks')
299330
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -303,6 +334,7 @@ def test_benchmarks_with_remainder_after_doubleminus(self):
303334
def test_filters_basic(self):
304335
parsed = self.parser.parse_args(
305336
['filters', self.testInput0, 'c', 'd'])
337+
self.assertFalse(parsed.display_aggregates_only)
306338
self.assertTrue(parsed.utest)
307339
self.assertEqual(parsed.mode, 'filters')
308340
self.assertEqual(parsed.test[0].name, self.testInput0)
@@ -313,6 +345,7 @@ def test_filters_basic(self):
313345
def test_filters_with_remainder(self):
314346
parsed = self.parser.parse_args(
315347
['filters', self.testInput0, 'c', 'd', 'e'])
348+
self.assertFalse(parsed.display_aggregates_only)
316349
self.assertTrue(parsed.utest)
317350
self.assertEqual(parsed.mode, 'filters')
318351
self.assertEqual(parsed.test[0].name, self.testInput0)
@@ -323,6 +356,7 @@ def test_filters_with_remainder(self):
323356
def test_filters_with_remainder_after_doubleminus(self):
324357
parsed = self.parser.parse_args(
325358
['filters', self.testInput0, 'c', 'd', '--', 'f'])
359+
self.assertFalse(parsed.display_aggregates_only)
326360
self.assertTrue(parsed.utest)
327361
self.assertEqual(parsed.mode, 'filters')
328362
self.assertEqual(parsed.test[0].name, self.testInput0)
@@ -333,6 +367,7 @@ def test_filters_with_remainder_after_doubleminus(self):
333367
def test_benchmarksfiltered_basic(self):
334368
parsed = self.parser.parse_args(
335369
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e'])
370+
self.assertFalse(parsed.display_aggregates_only)
336371
self.assertTrue(parsed.utest)
337372
self.assertEqual(parsed.mode, 'benchmarksfiltered')
338373
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -344,6 +379,7 @@ def test_benchmarksfiltered_basic(self):
344379
def test_benchmarksfiltered_with_remainder(self):
345380
parsed = self.parser.parse_args(
346381
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', 'f'])
382+
self.assertFalse(parsed.display_aggregates_only)
347383
self.assertTrue(parsed.utest)
348384
self.assertEqual(parsed.mode, 'benchmarksfiltered')
349385
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -355,6 +391,7 @@ def test_benchmarksfiltered_with_remainder(self):
355391
def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
356392
parsed = self.parser.parse_args(
357393
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', '--', 'g'])
394+
self.assertFalse(parsed.display_aggregates_only)
358395
self.assertTrue(parsed.utest)
359396
self.assertEqual(parsed.mode, 'benchmarksfiltered')
360397
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
@@ -365,7 +402,7 @@ def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
365402

366403

367404
if __name__ == '__main__':
368-
# unittest.main()
405+
#unittest.main()
369406
main()
370407

371408
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4

‎tools/gbench/Inputs/test3_run0.json

+26
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"benchmarks": [
1010
{
1111
"name": "BM_One",
12+
"run_type": "aggregate",
1213
"iterations": 1000,
1314
"real_time": 10,
1415
"cpu_time": 100,
@@ -25,15 +26,40 @@
2526
"name": "BM_Two",
2627
"iterations": 1000,
2728
"real_time": 8,
29+
"cpu_time": 86,
30+
"time_unit": "ns"
31+
},
32+
{
33+
"name": "short",
34+
"run_type": "aggregate",
35+
"iterations": 1000,
36+
"real_time": 8,
2837
"cpu_time": 80,
2938
"time_unit": "ns"
3039
},
3140
{
3241
"name": "short",
42+
"run_type": "aggregate",
43+
"iterations": 1000,
44+
"real_time": 8,
45+
"cpu_time": 77,
46+
"time_unit": "ns"
47+
},
48+
{
49+
"name": "medium",
50+
"run_type": "iteration",
3351
"iterations": 1000,
3452
"real_time": 8,
3553
"cpu_time": 80,
3654
"time_unit": "ns"
55+
},
56+
{
57+
"name": "medium",
58+
"run_type": "iteration",
59+
"iterations": 1000,
60+
"real_time": 9,
61+
"cpu_time": 82,
62+
"time_unit": "ns"
3763
}
3864
]
3965
}

‎tools/gbench/Inputs/test3_run1.json

+29-3
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
},
1717
{
1818
"name": "BM_Two",
19+
"run_type": "aggregate",
1920
"iterations": 1000,
2021
"real_time": 10,
2122
"cpu_time": 89,
@@ -25,14 +26,39 @@
2526
"name": "BM_Two",
2627
"iterations": 1000,
2728
"real_time": 7,
28-
"cpu_time": 70,
29+
"cpu_time": 72,
2930
"time_unit": "ns"
3031
},
3132
{
3233
"name": "short",
34+
"run_type": "aggregate",
3335
"iterations": 1000,
34-
"real_time": 8,
35-
"cpu_time": 80,
36+
"real_time": 7,
37+
"cpu_time": 75,
38+
"time_unit": "ns"
39+
},
40+
{
41+
"name": "short",
42+
"run_type": "aggregate",
43+
"iterations": 762,
44+
"real_time": 4.54,
45+
"cpu_time": 66.6,
46+
"time_unit": "ns"
47+
},
48+
{
49+
"name": "short",
50+
"run_type": "iteration",
51+
"iterations": 1000,
52+
"real_time": 800,
53+
"cpu_time": 1,
54+
"time_unit": "ns"
55+
},
56+
{
57+
"name": "medium",
58+
"run_type": "iteration",
59+
"iterations": 1200,
60+
"real_time": 5,
61+
"cpu_time": 53,
3662
"time_unit": "ns"
3763
}
3864
]

0 commit comments

Comments
 (0)