Merge pull request #238 from launchdarkly/eb/ch76240/benchmarks-4.x

(4.x) implement JMH benchmarks

Author: eli-darkly

.circleci/config.yml

@@ -30,6 +30,9 @@ workflows:
       - packaging:
           requires:
             - build-linux
+      - benchmarks:
+          requires:
+            - build-linux
       - build-test-windows:
           name: Java 11 - Windows - OpenJDK
 
@@ -131,3 +134,19 @@ jobs:
       - run:
           name: run packaging tests
           command: cd packaging-test && make all
+
+  benchmarks:
+    docker:
+      - image: circleci/openjdk:11
+    steps:
+      - run: java -version
+      - run: sudo apt-get install make -y -q
+      - checkout
+      - attach_workspace:
+          at: build
+      - run: cat gradle.properties.example >>gradle.properties
+      - run:
+          name: run benchmarks
+          command: cd benchmarks && make
+      - store_artifacts:
+          path: benchmarks/build/reports/jmh

.gitignore

@@ -17,3 +17,4 @@ out/
 classes/
 
 packaging-test/temp/
+benchmarks/lib/

CONTRIBUTING.md

@@ -42,3 +42,7 @@ To build the SDK and run all unit tests:
 ```
 
 By default, the full unit test suite includes live tests of the Redis integration. Those tests expect you to have Redis running locally. To skip them, set the environment variable `LD_SKIP_DATABASE_TESTS=1` before running the tests.
+
+### Benchmarks
+
+The project in the `benchmarks` subdirectory uses [JMH](https://openjdk.java.net/projects/code-tools/jmh/) to generate performance metrics for the SDK. This is run as a CI job, and can also be run manually by running `make` within `benchmarks` and then inspecting `build/reports/jmh`.

benchmarks/Makefile

@@ -0,0 +1,36 @@
+.PHONY: benchmark clean sdk
+
+BASE_DIR:=$(shell pwd)
+PROJECT_DIR=$(shell cd .. && pwd)
+SDK_VERSION=$(shell grep "version=" $(PROJECT_DIR)/gradle.properties | cut -d '=' -f 2)
+
+BENCHMARK_ALL_JAR=lib/launchdarkly-java-server-sdk-all.jar
+BENCHMARK_TEST_JAR=lib/launchdarkly-java-server-sdk-test.jar
+SDK_JARS_DIR=$(PROJECT_DIR)/build/libs
+SDK_ALL_JAR=$(SDK_JARS_DIR)/launchdarkly-java-server-sdk-$(SDK_VERSION)-all.jar
+SDK_TEST_JAR=$(SDK_JARS_DIR)/launchdarkly-java-server-sdk-$(SDK_VERSION)-test.jar
+
+benchmark: $(BENCHMARK_ALL_JAR) $(BENCHMARK_TEST_JAR)
+	rm -rf build/tmp
+	../gradlew jmh
+	cat build/reports/jmh/human.txt
+	../gradlew jmhReport
+
+clean:
+	rm -rf build lib
+
+sdk: $(BENCHMARK_ALL_JAR) $(BENCHMARK_TEST_JAR)
+
+$(BENCHMARK_ALL_JAR): $(SDK_ALL_JAR)
+	mkdir -p lib
+	cp $< $@
+
+$(BENCHMARK_TEST_JAR): $(SDK_TEST_JAR)
+	mkdir -p lib
+	cp $< $@
+
+$(SDK_ALL_JAR):
+	cd .. && ./gradlew shadowJarAll
+
+$(SDK_TEST_JAR):
+	cd .. && ./gradlew testJar

benchmarks/build.gradle

@@ -0,0 +1,63 @@
+
+buildscript {
+    repositories {
+        jcenter()
+        mavenCentral()
+    }
+}
+
+plugins {
+    id "me.champeau.gradle.jmh" version "0.5.0"
+    id "io.morethan.jmhreport" version "0.9.0"
+}
+
+repositories {
+    mavenCentral()
+}
+
+ext.versions = [
+    "jmh": "1.21",
+    "guava": "19.0"
+]
+
+dependencies {
+    compile files("lib/launchdarkly-java-server-sdk-all.jar")
+    compile files("lib/launchdarkly-java-server-sdk-test.jar")
+    compile "com.google.guava:guava:${versions.guava}" // required by SDK test code
+    compile "com.squareup.okhttp3:mockwebserver:3.12.10"
+    compile "org.openjdk.jmh:jmh-core:1.21"
+    compile "org.openjdk.jmh:jmh-generator-annprocess:${versions.jmh}"
+}
+
+jmh {
+    iterations = 10 // Number of measurement iterations to do.
+    benchmarkMode = ['avgt'] // "average time" - reports execution time as ns/op and allocations as B/op.
+    // batchSize = 1 // Batch size: number of benchmark method calls per operation. (some benchmark modes can ignore this setting)
+    fork = 1 // How many times to forks a single benchmark. Use 0 to disable forking altogether
+    // failOnError = false // Should JMH fail immediately if any benchmark had experienced the unrecoverable error?
+    forceGC = true // Should JMH force GC between iterations?
+    humanOutputFile = project.file("${project.buildDir}/reports/jmh/human.txt") // human-readable output file
+    // resultsFile = project.file("${project.buildDir}/reports/jmh/results.txt") // results file
+    operationsPerInvocation = 3 // Operations per invocation.
+    // benchmarkParameters =  [:] // Benchmark parameters.
+    profilers = [ 'gc' ] // Use profilers to collect additional data. Supported profilers: [cl, comp, gc, stack, perf, perfnorm, perfasm, xperf, xperfasm, hs_cl, hs_comp, hs_gc, hs_rt, hs_thr]
+    timeOnIteration = '1s' // Time to spend at each measurement iteration.
+    resultFormat = 'JSON' // Result format type (one of CSV, JSON, NONE, SCSV, TEXT)
+    // synchronizeIterations = false // Synchronize iterations?
+    // threads = 4 // Number of worker threads to run with.
+    // timeout = '1s' // Timeout for benchmark iteration.
+    timeUnit = 'ns' // Output time unit. Available time units are: [m, s, ms, us, ns].
+    verbosity = 'NORMAL' // Verbosity mode. Available modes are: [SILENT, NORMAL, EXTRA]
+    warmup = '1s' // Time to spend at each warmup iteration.
+    warmupBatchSize = 2 // Warmup batch size: number of benchmark method calls per operation.
+    warmupIterations = 1 // Number of warmup iterations to do.
+    // warmupForks = 0 // How many warmup forks to make for a single benchmark. 0 to disable warmup forks.
+    // warmupMode = 'INDI' // Warmup mode for warming up selected benchmarks. Warmup modes are: [INDI, BULK, BULK_INDI].
+
+    jmhVersion = versions.jmh
+}
+
+jmhReport {
+    jmhResultPath = project.file('build/reports/jmh/results.json')
+    jmhReportOutput = project.file('build/reports/jmh')
+}

benchmarks/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = 'launchdarkly-java-server-sdk-benchmarks'

benchmarks/src/jmh/java/com/launchdarkly/client/EventProcessorInternals.java

@@ -0,0 +1,14 @@
+package com.launchdarkly.client;
+
+import java.io.IOException;
+
+// Placed here so we can access package-private SDK methods.
+public class EventProcessorInternals {
+  public static void waitUntilInactive(EventProcessor ep) {
+    try {
+      ((DefaultEventProcessor)ep).waitUntilInactive();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

benchmarks/src/jmh/java/com/launchdarkly/client/FlagData.java

@@ -0,0 +1,82 @@
+package com.launchdarkly.client;
+
+import com.launchdarkly.client.value.LDValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+
+import static com.launchdarkly.client.TestUtil.fallthroughVariation;
+import static com.launchdarkly.client.TestUtil.flagWithValue;
+import static com.launchdarkly.client.VersionedDataKind.FEATURES;
+import static com.launchdarkly.sdk.server.TestValues.BOOLEAN_FLAG_KEY;
+import static com.launchdarkly.sdk.server.TestValues.CLAUSE_MATCH_ATTRIBUTE;
+import static com.launchdarkly.sdk.server.TestValues.CLAUSE_MATCH_VALUES;
+import static com.launchdarkly.sdk.server.TestValues.FLAG_WITH_MULTI_VALUE_CLAUSE_KEY;
+import static com.launchdarkly.sdk.server.TestValues.FLAG_WITH_PREREQ_KEY;
+import static com.launchdarkly.sdk.server.TestValues.FLAG_WITH_TARGET_LIST_KEY;
+import static com.launchdarkly.sdk.server.TestValues.INT_FLAG_KEY;
+import static com.launchdarkly.sdk.server.TestValues.JSON_FLAG_KEY;
+import static com.launchdarkly.sdk.server.TestValues.STRING_FLAG_KEY;
+import static com.launchdarkly.sdk.server.TestValues.TARGETED_USER_KEYS;
+
+// This class must be in com.launchdarkly.client because FeatureFlagBuilder is package-private in the
+// SDK, but we are keeping the rest of the benchmark implementation code in com.launchdarkly.sdk.server
+// so we can more clearly compare between 4.x and 5.0.
+public class FlagData {
+  public static void loadTestFlags(FeatureStore store) {
+    for (FeatureFlag flag: FlagData.makeTestFlags()) {
+      store.upsert(FEATURES, flag);
+    }
+  }
+  
+  public static List<FeatureFlag> makeTestFlags() {
+    List<FeatureFlag> flags = new ArrayList<>();
+
+    flags.add(flagWithValue(BOOLEAN_FLAG_KEY, LDValue.of(true)));
+    flags.add(flagWithValue(INT_FLAG_KEY, LDValue.of(1)));
+    flags.add(flagWithValue(STRING_FLAG_KEY, LDValue.of("x")));
+    flags.add(flagWithValue(JSON_FLAG_KEY, LDValue.buildArray().build()));
+
+    FeatureFlag targetsFlag = new FeatureFlagBuilder(FLAG_WITH_TARGET_LIST_KEY)
+      .on(true)
+      .targets(Arrays.asList(new Target(new HashSet<String>(TARGETED_USER_KEYS), 1)))
+      .fallthrough(fallthroughVariation(0))
+      .offVariation(0)
+      .variations(LDValue.of(false), LDValue.of(true))
+      .build();
+    flags.add(targetsFlag);
+
+    FeatureFlag prereqFlag = new FeatureFlagBuilder("prereq-flag")
+      .on(true)
+      .fallthrough(fallthroughVariation(1))
+      .variations(LDValue.of(false), LDValue.of(true))
+      .build();
+    flags.add(prereqFlag);
+
+    FeatureFlag flagWithPrereq = new FeatureFlagBuilder(FLAG_WITH_PREREQ_KEY)
+      .on(true)
+      .prerequisites(Arrays.asList(new Prerequisite("prereq-flag", 1)))
+      .fallthrough(fallthroughVariation(1))
+      .offVariation(0)
+      .variations(LDValue.of(false), LDValue.of(true))
+      .build();
+    flags.add(flagWithPrereq);
+
+    FeatureFlag flagWithMultiValueClause = new FeatureFlagBuilder(FLAG_WITH_MULTI_VALUE_CLAUSE_KEY)
+      .on(true)
+      .fallthrough(fallthroughVariation(0))
+      .offVariation(0)
+      .variations(LDValue.of(false), LDValue.of(true))
+      .rules(Arrays.asList(
+          new RuleBuilder()
+            .clauses(new Clause(CLAUSE_MATCH_ATTRIBUTE, Operator.in, CLAUSE_MATCH_VALUES, false))
+            .build()
+          ))
+      .build();
+    flags.add(flagWithMultiValueClause);
+    
+    return flags;
+  }
+}

benchmarks/src/jmh/java/com/launchdarkly/sdk/server/EventProcessorBenchmarks.java

@@ -0,0 +1,137 @@
+package com.launchdarkly.sdk.server;
+
+import com.launchdarkly.client.Components;
+import com.launchdarkly.client.Event;
+import com.launchdarkly.client.EventProcessor;
+import com.launchdarkly.client.EventProcessorInternals;
+import com.launchdarkly.client.LDConfig;
+import com.launchdarkly.client.interfaces.EventSender;
+import com.launchdarkly.client.interfaces.EventSenderFactory;
+import com.launchdarkly.client.interfaces.HttpConfiguration;
+import com.launchdarkly.client.value.LDValue;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import static com.launchdarkly.sdk.server.TestValues.BASIC_USER;
+import static com.launchdarkly.sdk.server.TestValues.CUSTOM_EVENT;
+import static com.launchdarkly.sdk.server.TestValues.TEST_EVENTS_COUNT;
+
+public class EventProcessorBenchmarks {
+  private static final int EVENT_BUFFER_SIZE = 1000;
+  private static final int FLAG_COUNT = 10;
+  private static final int FLAG_VERSIONS = 3;
+  private static final int FLAG_VARIATIONS = 2;
+  
+  @State(Scope.Thread)
+  public static class BenchmarkInputs {
+    // Initialization of the things in BenchmarkInputs does not count as part of a benchmark.
+    final EventProcessor eventProcessor;
+    final EventSender eventSender;
+    final List<Event.FeatureRequest> featureRequestEventsWithoutTracking = new ArrayList<>();
+    final List<Event.FeatureRequest> featureRequestEventsWithTracking = new ArrayList<>();
+    final Random random;
+
+    public BenchmarkInputs() {
+      // MockEventSender does no I/O - it discards every event payload. So we are benchmarking
+      // all of the event processing steps up to that point, including the formatting of the
+      // JSON data in the payload.
+      eventSender = new MockEventSender();
+      
+      eventProcessor = Components.sendEvents()
+          .capacity(EVENT_BUFFER_SIZE)
+          .eventSender(new MockEventSenderFactory())
+          .createEventProcessor(TestValues.SDK_KEY, new LDConfig.Builder().build());
+      
+      random = new Random();
+      
+      for (int i = 0; i < TEST_EVENTS_COUNT; i++) {
+        String flagKey = "flag" + random.nextInt(FLAG_COUNT);
+        int version = random.nextInt(FLAG_VERSIONS) + 1;
+        int variation = random.nextInt(FLAG_VARIATIONS);
+        for (boolean trackEvents: new boolean[] { false, true }) {
+          Event.FeatureRequest event = new Event.FeatureRequest(
+              System.currentTimeMillis(),
+              flagKey,
+              BASIC_USER,
+              version,
+              variation,
+              LDValue.of(variation),
+              LDValue.ofNull(),
+              null,
+              null,
+              trackEvents,
+              null,
+              false
+              );
+          (trackEvents ? featureRequestEventsWithTracking : featureRequestEventsWithoutTracking).add(event);
+        }
+      }
+    }
+    
+    public String randomFlagKey() {
+      return "flag" + random.nextInt(FLAG_COUNT);
+    }
+    
+    public int randomFlagVersion() {
+      return random.nextInt(FLAG_VERSIONS) + 1;
+    }
+    
+    public int randomFlagVariation() {
+      return random.nextInt(FLAG_VARIATIONS);
+    }
+  }
+  
+  @Benchmark
+  public void summarizeFeatureRequestEvents(BenchmarkInputs inputs) throws Exception {
+    for (Event.FeatureRequest event: inputs.featureRequestEventsWithoutTracking) {
+      inputs.eventProcessor.sendEvent(event);
+    }
+    inputs.eventProcessor.flush();
+    EventProcessorInternals.waitUntilInactive(inputs.eventProcessor);
+  }
+
+  @Benchmark
+  public void featureRequestEventsWithFullTracking(BenchmarkInputs inputs) throws Exception {
+    for (Event.FeatureRequest event: inputs.featureRequestEventsWithTracking) {
+      inputs.eventProcessor.sendEvent(event);
+    }
+    inputs.eventProcessor.flush();
+    EventProcessorInternals.waitUntilInactive(inputs.eventProcessor);
+  }
+  
+  @Benchmark
+  public void customEvents(BenchmarkInputs inputs) throws Exception {
+    for (int i = 0; i < TEST_EVENTS_COUNT; i++) {
+      inputs.eventProcessor.sendEvent(CUSTOM_EVENT);
+    }
+    inputs.eventProcessor.flush();
+    EventProcessorInternals.waitUntilInactive(inputs.eventProcessor);
+  }
+  
+  private static final class MockEventSender implements EventSender {
+    private static final Result RESULT = new Result(true, false, null);
+    
+    @Override
+    public void close() throws IOException {}
+
+    @Override
+    public Result sendEventData(EventDataKind arg0, String arg1, int arg2, URI arg3) {
+      return RESULT;
+    }
+  }
+  
+  private static final class MockEventSenderFactory implements EventSenderFactory {
+    @Override
+    public EventSender createEventSender(String arg0, HttpConfiguration arg1) {
+      return new MockEventSender();
+    }
+  }
+}