Skip to content

Commit 21c901b

Browse files
author
Pap Lőrinc
committed
Unified benchmarks
1 parent 930163f commit 21c901b

File tree

5 files changed

+127
-110
lines changed

5 files changed

+127
-110
lines changed

javaslang-benchmark/src/test/java/javaslang/benchmark/BenchmarkPerformanceReporter.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -224,11 +224,11 @@ private String calculateRatioStr(TestExecution baseResult, Option<TestExecution>
224224
if (!alternativeResult.isDefined()) {
225225
return "";
226226
}
227-
double alternativeScore = alternativeResult.get().getScore();
227+
final double alternativeScore = alternativeResult.get().getScore();
228228
if (alternativeScore == 0.0) {
229229
return "";
230230
}
231-
double ratio = baseResult.getScore() / alternativeScore;
231+
final double ratio = baseResult.getScore() / alternativeScore;
232232
return ratio == 1.0 ? "" : PERFORMANCE_FORMAT.format(ratio) + "x";
233233
}
234234
}
@@ -390,14 +390,14 @@ public static TestExecution of(BenchmarkResult benchmarkResult, double outlierLo
390390
}
391391

392392
public TestExecution(BenchmarkResult benchmark) {
393-
Result primaryResult = benchmark.getPrimaryResult();
393+
final Result<?> primaryResult = benchmark.getPrimaryResult();
394394
fullName = benchmark.getParams().getBenchmark();
395395
target = extractPart(fullName, 2);
396396
operation = extractPart(fullName, 1);
397397
implementation = extractPart(fullName, 0);
398398
paramKey = getParameterKey(benchmark);
399399

400-
ListStatistics statistics = createStatisticsWithoutOutliers(benchmark, outlierLowPct, outlierHighPct);
400+
final ListStatistics statistics = createStatisticsWithoutOutliers(benchmark, outlierLowPct, outlierHighPct);
401401
sampleCount = statistics.getN();
402402
score = statistics.getMean();
403403
scoreError = statistics.getMeanErrorAt(0.999);
@@ -409,15 +409,15 @@ private ListStatistics createStatisticsWithoutOutliers(BenchmarkResult benchmark
409409
.map(r -> r.getPrimaryResult().getScore())
410410
.sorted()
411411
.collect(Vector.collector());
412-
int size = results.size();
413-
int outliersLow = (int) (size * outlierLowPct);
414-
int outliersHigh = (int) (size * outlierHighPct);
412+
final int size = results.size();
413+
final int outliersLow = (int) (size * outlierLowPct);
414+
final int outliersHigh = (int) (size * outlierHighPct);
415415
results = results.drop(outliersLow).dropRight(outliersHigh);
416416
return new ListStatistics(results.toJavaList().stream().mapToDouble(r -> r).toArray());
417417
}
418418

419419
private String getParameterKey(BenchmarkResult benchmarkResult) {
420-
BenchmarkParams params = benchmarkResult.getParams();
420+
final BenchmarkParams params = benchmarkResult.getParams();
421421
return params.getParamsKeys().stream().map(params::getParam).collect(Collectors.joining(","));
422422
}
423423

javaslang-benchmark/src/test/java/javaslang/benchmark/JmhRunner.java

Lines changed: 46 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,70 +3,95 @@
33
import org.openjdk.jmh.annotations.Mode;
44
import org.openjdk.jmh.results.RunResult;
55
import org.openjdk.jmh.runner.Runner;
6-
import org.openjdk.jmh.runner.options.Options;
7-
import org.openjdk.jmh.runner.options.OptionsBuilder;
8-
import org.openjdk.jmh.runner.options.TimeValue;
6+
import org.openjdk.jmh.runner.options.*;
97

10-
import java.util.Collection;
8+
import java.util.*;
119
import java.util.concurrent.TimeUnit;
1210

1311
public class JmhRunner {
14-
private static final int WARMUP_ITERATIONS = 10;
15-
private static final int MEASUREMENT_ITERATIONS = 40;
12+
private static final int WARMUP_ITERATIONS = 20;
13+
private static final int MEASUREMENT_ITERATIONS = 30;
1614

17-
private static final int QUICK_WARMUP_ITERATIONS = 5;
15+
private static final int QUICK_WARMUP_ITERATIONS = 10;
1816
private static final int QUICK_MEASUREMENT_ITERATIONS = 10;
1917

2018
public static void run(Class<?> benchmarkClass) {
21-
runAndReport(benchmarkClass, WARMUP_ITERATIONS, MEASUREMENT_ITERATIONS, Assertions.Disable);
19+
runAndReport(benchmarkClass, WARMUP_ITERATIONS, MEASUREMENT_ITERATIONS, 500, PrintGc.Enable, Assertions.Disable);
2220
}
2321

2422
public static void devRun(Class<?> benchmarkClass) {
25-
runAndReport(benchmarkClass, QUICK_WARMUP_ITERATIONS, QUICK_MEASUREMENT_ITERATIONS, Assertions.Disable);
23+
runAndReport(benchmarkClass, QUICK_WARMUP_ITERATIONS, QUICK_MEASUREMENT_ITERATIONS, 200, PrintGc.Disable, Assertions.Disable);
2624
}
2725

2826
public static void devRunWithAssertions(Class<?> benchmarkClass) {
29-
runAndReport(benchmarkClass, QUICK_WARMUP_ITERATIONS, QUICK_MEASUREMENT_ITERATIONS, Assertions.Enable);
27+
runAndReport(benchmarkClass, QUICK_WARMUP_ITERATIONS, QUICK_MEASUREMENT_ITERATIONS, 200, PrintGc.Disable, Assertions.Enable);
3028
}
3129

32-
private static void runAndReport(Class<?> benchmarkClass, int warmupIterations, int measurementIterations, Assertions assertions) {
33-
Collection<RunResult> results = run(benchmarkClass, warmupIterations, measurementIterations, assertions);
30+
private static void runAndReport(Class<?> benchmarkClass, int warmupIterations, int measurementIterations, int millis, PrintGc printGc, Assertions assertions) {
31+
final Collection<RunResult> results = run(benchmarkClass, warmupIterations, measurementIterations, millis, printGc, assertions);
3432
BenchmarkPerformanceReporter.of(results).print();
3533
}
3634

37-
private static Collection<RunResult> run(Class<?> benchmarkClass, int warmupIterations, int measurementIterations, Assertions assertions) {
35+
private static Collection<RunResult> run(Class<?> benchmarkClass, int warmupIterations, int measurementIterations, int millis, PrintGc printGc, Assertions assertions) {
3836
final Options opts = new OptionsBuilder()
3937
.include(benchmarkClass.getSimpleName())
4038
.shouldDoGC(true)
4139
.shouldFailOnError(true)
4240
.mode(Mode.Throughput)
4341
.timeUnit(TimeUnit.SECONDS)
44-
.warmupTime(TimeValue.milliseconds(500))
42+
.warmupTime(TimeValue.milliseconds(millis))
4543
.warmupIterations(warmupIterations)
46-
.measurementTime(TimeValue.milliseconds(500))
44+
.measurementTime(TimeValue.milliseconds(millis))
4745
.measurementIterations(measurementIterations)
4846
.forks(1)
4947
// We are using 4Gb and setting NewGen to 100% to avoid GC during testing.
5048
// Any GC during testing will destroy the iteration, which should get ignored as an outlier
51-
.jvmArgsAppend("-XX:+UseG1GC", "-Xss100m", "-Xms4g", "-Xmx4g", "-XX:+PrintGC", "-XX:MaxGCPauseMillis=1000", "-XX:+UnlockExperimentalVMOptions", "-XX:G1NewSizePercent=100", "-XX:G1MaxNewSizePercent=100", assertions.vmArg)
49+
.jvmArgsAppend("-XX:+UseG1GC", "-Xss100m", "-Xms4g", "-Xmx4g", "-XX:MaxGCPauseMillis=1000", "-XX:+UnlockExperimentalVMOptions", "-XX:G1NewSizePercent=100", "-XX:G1MaxNewSizePercent=100", printGc.vmArg, assertions.vmArg)
5250
.build();
5351

5452
try {
5553
return new Runner(opts).run();
56-
}
57-
catch (Exception e) {
54+
} catch (Exception e) {
5855
throw new RuntimeException(e.getMessage(), e);
5956
}
6057
}
6158

62-
private static enum Assertions {
59+
private enum Assertions {
6360
Enable("-enableassertions"),
64-
Disable("-disableassertions")
65-
;
61+
Disable("-disableassertions");
6662

6763
final String vmArg;
64+
6865
Assertions(String vmArg) {
6966
this.vmArg = vmArg;
7067
}
7168
}
69+
70+
private enum PrintGc {
71+
Enable("-XX:+PrintGC"),
72+
Disable("-XX:-PrintGC");
73+
74+
final String vmArg;
75+
76+
PrintGc(String vmArg) {
77+
this.vmArg = vmArg;
78+
}
79+
}
80+
81+
public static <T> void assertEquals(T a, T b) {
82+
if (!Objects.equals(a, b)) {
83+
throw new IllegalStateException(a + " != " + b);
84+
}
85+
}
86+
87+
public static Integer[] getRandomValues(int size, int seed) {
88+
final Random random = new Random(seed);
89+
90+
final Integer[] results = new Integer[size];
91+
for (int i = 0; i < size; i++) {
92+
final int value = random.nextInt(size) - (size / 2);
93+
results[i] = value;
94+
}
95+
return results;
96+
}
7297
}

javaslang-benchmark/src/test/java/javaslang/benchmark/collection/ArrayBenchmark.java

Lines changed: 6 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,25 @@
11
package javaslang.benchmark.collection;
22

3-
import javaslang.benchmark.JmhRunner;
4-
import org.openjdk.jmh.annotations.Benchmark;
5-
import org.openjdk.jmh.annotations.Level;
6-
import org.openjdk.jmh.annotations.Param;
7-
import org.openjdk.jmh.annotations.Scope;
8-
import org.openjdk.jmh.annotations.Setup;
9-
import org.openjdk.jmh.annotations.State;
10-
import org.openjdk.jmh.annotations.TearDown;
11-
12-
import java.util.Objects;
13-
import java.util.Random;
3+
import org.openjdk.jmh.annotations.*;
4+
5+
import static javaslang.benchmark.JmhRunner.*;
146

157
public class ArrayBenchmark {
168

179
public static void main(String... args) { /* main is more reliable than a test */
18-
JmhRunner.run(ArrayBenchmark.class);
10+
run(ArrayBenchmark.class);
1911
}
2012

2113
@State(Scope.Benchmark)
2214
public static class Base {
23-
@Param({ "10", "100", "1000", "10000"})
15+
@Param({ "10", "100", "1000", "10000" })
2416
public int CONTAINER_SIZE;
2517

2618
public Integer[] ELEMENTS;
2719

2820
@Setup
2921
public void setup() {
30-
final Random random = new Random(0);
31-
32-
ELEMENTS = new Integer[CONTAINER_SIZE];
33-
for (int i = 0; i < CONTAINER_SIZE; i++) {
34-
final int value = random.nextInt(CONTAINER_SIZE) - (CONTAINER_SIZE / 2);
35-
ELEMENTS[i] = value;
36-
}
37-
}
38-
39-
protected static <T> void assertEquals(T a, T b) {
40-
if (!Objects.equals(a, b)) {
41-
throw new IllegalStateException(a + " != " + b);
42-
}
22+
ELEMENTS = getRandomValues(CONTAINER_SIZE, 0);
4323
}
4424
}
4525

javaslang-benchmark/src/test/java/javaslang/benchmark/collection/ListBenchmark.java

Lines changed: 58 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,28 @@
11
package javaslang.benchmark.collection;
22

3-
import javaslang.benchmark.JmhRunner;
4-
import org.openjdk.jmh.annotations.Benchmark;
5-
import org.openjdk.jmh.annotations.Level;
6-
import org.openjdk.jmh.annotations.Param;
7-
import org.openjdk.jmh.annotations.Scope;
8-
import org.openjdk.jmh.annotations.Setup;
9-
import org.openjdk.jmh.annotations.State;
10-
import org.openjdk.jmh.annotations.TearDown;
11-
12-
import java.util.Collections;
13-
import java.util.Iterator;
14-
import java.util.Objects;
15-
import java.util.Random;
3+
import org.openjdk.jmh.annotations.*;
4+
import scala.compat.java8.JFunction;
165

17-
public class ListBenchmark {
6+
import java.util.*;
7+
import java.util.stream.Collectors;
8+
9+
import static javaslang.benchmark.JmhRunner.*;
1810

11+
public class ListBenchmark {
1912
public static void main(String... args) { /* main is more reliable than a test */
20-
JmhRunner.run(ListBenchmark.class);
13+
run(ListBenchmark.class);
2114
}
2215

2316
@State(Scope.Benchmark)
2417
public static class Base {
25-
@Param({ "10", "100", "1000", "10000"})
18+
@Param({ "10", "100", "1000" })
2619
public int CONTAINER_SIZE;
2720

2821
public Integer[] ELEMENTS;
2922

3023
@Setup
3124
public void setup() {
32-
final Random random = new Random(0);
33-
34-
ELEMENTS = new Integer[CONTAINER_SIZE];
35-
for (int i = 0; i < CONTAINER_SIZE; i++) {
36-
final int value = random.nextInt(CONTAINER_SIZE) - (CONTAINER_SIZE / 2);
37-
ELEMENTS[i] = value;
38-
}
39-
}
40-
41-
protected static <T> void assertEquals(T a, T b) {
42-
if (!Objects.equals(a, b)) {
43-
throw new IllegalStateException(a + " != " + b);
44-
}
25+
ELEMENTS = getRandomValues(CONTAINER_SIZE, 0);
4526
}
4627
}
4728

@@ -240,4 +221,52 @@ public void slang_persistent(Initialized state) {
240221
}
241222
}
242223

224+
public static class GroupBy extends Base {
225+
@State(Scope.Thread)
226+
public static class Initialized {
227+
final java.util.ArrayList<Integer> javaMutable = new java.util.ArrayList<>();
228+
scala.collection.immutable.List<Integer> scalaPersistent = scala.collection.immutable.List$.MODULE$.empty();
229+
fj.data.List<Integer> fjavaPersistent = fj.data.List.list();
230+
javaslang.collection.List<Integer> slangPersistent = javaslang.collection.List.empty();
231+
232+
@Setup
233+
public void initializeMutable(Base state) {
234+
assertEquals(javaMutable.size(), 0);
235+
Collections.addAll(javaMutable, state.ELEMENTS);
236+
assertEquals(javaMutable.size(), state.CONTAINER_SIZE);
237+
238+
assertEquals(fjavaPersistent.length(), 0);
239+
assertEquals(scalaPersistent.size(), 0);
240+
assertEquals(slangPersistent.size(), 0);
241+
for (Integer element : state.ELEMENTS) {
242+
fjavaPersistent = fjavaPersistent.cons(element);
243+
scalaPersistent = scalaPersistent.$colon$colon(element);
244+
slangPersistent = slangPersistent.prepend(element);
245+
}
246+
assertEquals(fjavaPersistent.length(), state.CONTAINER_SIZE);
247+
assertEquals(scalaPersistent.size(), state.CONTAINER_SIZE);
248+
assertEquals(slangPersistent.size(), state.CONTAINER_SIZE);
249+
}
250+
}
251+
252+
@Benchmark
253+
public Object java_mutable(Initialized state) {
254+
return state.javaMutable.stream().collect(Collectors.groupingBy(Integer::bitCount));
255+
}
256+
257+
@Benchmark
258+
public Object scala_persistent(Initialized state) {
259+
return state.scalaPersistent.groupBy(JFunction.func(Integer::bitCount));
260+
}
261+
262+
@Benchmark
263+
public Object fjava_persistent(Initialized state) {
264+
return state.fjavaPersistent.groupBy(Integer::bitCount);
265+
}
266+
267+
@Benchmark
268+
public Object slang_persistent(Initialized state) {
269+
return state.slangPersistent.groupBy(Integer::bitCount);
270+
}
271+
}
243272
}

javaslang-benchmark/src/test/java/javaslang/benchmark/collection/PriorityQueueBenchmark.java

Lines changed: 9 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,40 @@
11
package javaslang.benchmark.collection;
22

33
import javaslang.Tuple2;
4-
import javaslang.benchmark.JmhRunner;
54
import javaslang.collection.Traversable;
6-
import org.openjdk.jmh.annotations.Benchmark;
7-
import org.openjdk.jmh.annotations.Level;
8-
import org.openjdk.jmh.annotations.Param;
9-
import org.openjdk.jmh.annotations.Scope;
10-
import org.openjdk.jmh.annotations.Setup;
5+
import org.openjdk.jmh.annotations.*;
116
import org.openjdk.jmh.annotations.State;
12-
import org.openjdk.jmh.annotations.TearDown;
137
import scala.math.Ordering;
148
import scala.math.Ordering$;
15-
import scalaz.Heap;
16-
import scalaz.Order;
17-
import scalaz.Order$;
9+
import scalaz.*;
1810

1911
import java.util.Collections;
20-
import java.util.Objects;
21-
import java.util.Random;
12+
13+
import static javaslang.benchmark.JmhRunner.*;
2214

2315
public class PriorityQueueBenchmark {
2416

2517
public static void main(String... args) { /* main is more reliable than a test */
26-
JmhRunner.run(PriorityQueueBenchmark.class);
18+
run(PriorityQueueBenchmark.class);
2719
}
2820

2921
@State(Scope.Benchmark)
3022
public static class Base {
3123
protected static final Ordering<Integer> SCALA_ORDERING = Ordering$.MODULE$.comparatorToOrdering(Integer::compareTo);
3224
protected static final Order<Integer> SCALAZ_ORDER = Order$.MODULE$.fromScalaOrdering(SCALA_ORDERING);
3325

34-
@Param({ "10", "100", "1000", "10000"})
26+
@Param({ "10", "100", "1000", "10000" })
3527
public int CONTAINER_SIZE;
3628

3729
public Integer[] ELEMENTS;
3830
int expectedAggregate = 0;
3931

4032
@Setup
4133
public void setup() {
42-
final Random random = new Random(0);
43-
44-
ELEMENTS = new Integer[CONTAINER_SIZE];
45-
for (int i = 0; i < CONTAINER_SIZE; i++) {
46-
final int value = random.nextInt(CONTAINER_SIZE) - (CONTAINER_SIZE / 2);
47-
ELEMENTS[i] = value;
48-
expectedAggregate ^= value;
49-
}
50-
}
34+
ELEMENTS = getRandomValues(CONTAINER_SIZE, 0);
5135

52-
protected static <T> void assertEquals(T a, T b) {
53-
if (!Objects.equals(a, b)) {
54-
throw new IllegalStateException(a + " != " + b);
36+
for (int element : ELEMENTS) {
37+
expectedAggregate ^= element;
5538
}
5639
}
5740
}

0 commit comments

Comments
 (0)