Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore: minor improvements to default benchmarks. #2993

Merged
merged 3 commits into from
Apr 2, 2024
Merged

chore: minor improvements to default benchmarks. #2993

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4b93495
chore: minor improvements to default benchmarks.
arpan14 Apr 2, 2024
5ec7c34
chore: lint issues fix.
arpan14 Apr 2, 2024
e83f463
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Apr 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 54 additions & 6 deletions google-cloud-spanner/src/test/java/com/google/cloud/spanner/AbstractLatencyBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

package com.google.cloud.spanner;

import com.google.common.base.MoreObjects;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
Expand All @@ -24,6 +25,48 @@

public abstract class AbstractLatencyBenchmark {

static final String SELECT_QUERY = "SELECT ID FROM FOO WHERE ID = @id";
static final String UPDATE_QUERY = "UPDATE FOO SET BAR=1 WHERE ID = @id";
static final String ID_COLUMN_NAME = "id";

/**
* Used to determine how many concurrent requests are allowed. For ex - To simulate a low QPS
* scenario, using 1 thread means there will be 1 request. Use a value > 1 to have concurrent
* requests.
*/
static final int PARALLEL_THREADS =
Integer.valueOf(MoreObjects.firstNonNull(
System.getenv("SPANNER_TEST_JMH_NUM_PARALLEL_THREADS"), "30"));

/**
* Total number of reads per test run for 1 thread. Increasing the value here will increase the
* duration of the benchmark. For ex - With PARALLEL_THREADS = 2, TOTAL_READS_PER_RUN = 200, there
* will be 400 read requests (200 on each thread).
*/
static final int TOTAL_READS_PER_RUN = Integer.valueOf(MoreObjects.firstNonNull(
System.getenv("SPANNER_TEST_JMH_NUM_READS_PER_THREAD"), "48000"));

/**
* Total number of writes per test run for 1 thread. Increasing the value here will increase the
* duration of the benchmark. For ex - With PARALLEL_THREADS = 2, TOTAL_WRITES_PER_RUN = 200,
* there will be 400 write requests (200 on each thread).
*/
static final int TOTAL_WRITES_PER_RUN = Integer.valueOf(MoreObjects.firstNonNull(
System.getenv("SPANNER_TEST_JMH_NUM_WRITES_PER_THREAD"), "4000"));

/**
* Number of requests which are used to initialise/warmup the benchmark. The latency number of
* these runs are ignored from the final reported results.
*/
static final int WARMUP_REQUEST_COUNT = 1;

/**
* Numbers of records in the sample table used in the benchmark. This is used in this benchmark to
* randomly choose a primary key and ensure that the reads are randomly distributed. This is done
* to ensure we don't end up reading/writing the same table record (leading to hot-spotting).
*/
static final int TOTAL_RECORDS = 1000000;

/** Utility to print latency numbers. It computes metrics such as Average, P50, P95 and P99. */
public void printResults(List<Duration> results) {
if (results == null) {
Expand All @@ -33,23 +76,23 @@ public void printResults(List<Duration> results) {
Collections.sort(orderedResults);
System.out.println();
System.out.printf("Total number of queries: %d\n", orderedResults.size());
System.out.printf("Avg: %fs\n", avg(results));
System.out.printf("P50: %fs\n", percentile(50, orderedResults));
System.out.printf("P95: %fs\n", percentile(95, orderedResults));
System.out.printf("P99: %fs\n", percentile(99, orderedResults));
System.out.printf("Avg: %fms\n", avg(results));
System.out.printf("P50: %fms\n", percentile(50, orderedResults));
System.out.printf("P95: %fms\n", percentile(95, orderedResults));
System.out.printf("P99: %fms\n", percentile(99, orderedResults));
}

private double percentile(int percentile, List<Duration> orderedResults) {
int index = percentile * orderedResults.size() / 100;
Duration value = orderedResults.get(index);
Double convertedValue = convertDurationToFractionInSeconds(value);
Double convertedValue = convertDurationToFractionInMilliSeconds(value);
return convertedValue;
}

/** Returns the average duration in seconds from a list of duration values. */
private double avg(List<Duration> results) {
return results.stream()
.collect(Collectors.averagingDouble(this::convertDurationToFractionInSeconds));
.collect(Collectors.averagingDouble(this::convertDurationToFractionInMilliSeconds));
}

private double convertDurationToFractionInSeconds(Duration duration) {
Expand All @@ -59,4 +102,9 @@ private double convertDurationToFractionInSeconds(Duration duration) {
double value = seconds + fraction;
return value;
}

private double convertDurationToFractionInMilliSeconds(Duration duration) {
long nanoseconds = duration.toNanos();
return nanoseconds / 1000000.0;
}
}
58 changes: 16 additions & 42 deletions google-cloud-spanner/src/test/java/com/google/cloud/spanner/DefaultBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

import static com.google.cloud.spanner.BenchmarkingUtilityScripts.collectResults;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;

import com.google.common.base.Stopwatch;
import com.google.common.util.concurrent.ListenableFuture;
Expand All @@ -37,6 +39,7 @@
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
Expand All @@ -45,8 +48,8 @@

/**
* Benchmarks for measuring existing latencies of various APIs using the Java Client. The benchmarks
* are bound to the Maven profile `benchmark` and can be executed like this: <code> mvn clean test
* -DskipTests -Pbenchmark -Dbenchmark.name=DefaultBenchmark
* are bound to the Maven profile `benchmark` and can be executed like this: <code>
* mvn clean test -DskipTests -Pbenchmark -Dbenchmark.name=DefaultBenchmark
* </code> Test Table Schema :
*
* <p>CREATE TABLE FOO ( id INT64 NOT NULL, BAZ INT64, BAR INT64, ) PRIMARY KEY(id);
Expand All @@ -63,47 +66,9 @@
@Fork(value = 1, warmups = 0)
@Measurement(batchSize = 1, iterations = 1, timeUnit = TimeUnit.MILLISECONDS)
@OutputTimeUnit(TimeUnit.SECONDS)
@Warmup(iterations = 1)
@Warmup(iterations = 0)
public class DefaultBenchmark extends AbstractLatencyBenchmark {

private static final String SELECT_QUERY = "SELECT ID FROM FOO WHERE ID = @id";
private static final String UPDATE_QUERY = "UPDATE FOO SET BAR=1 WHERE ID = @id";
private static final String ID_COLUMN_NAME = "id";

/**
* Used to determine how many concurrent requests are allowed. For ex - To simulate a low QPS
* scenario, using 1 thread means there will be 1 request. Use a value > 1 to have concurrent
* requests.
*/
private static final int PARALLEL_THREADS = 1;

/**
* Total number of reads per test run for 1 thread. Increasing the value here will increase the
* duration of the benchmark. For ex - With PARALLEL_THREADS = 2, TOTAL_READS_PER_RUN = 200, there
* will be 400 read requests (200 on each thread).
*/
private static final int TOTAL_READS_PER_RUN = 12000;

/**
* Total number of writes per test run for 1 thread. Increasing the value here will increase the
* duration of the benchmark. For ex - With PARALLEL_THREADS = 2, TOTAL_WRITES_PER_RUN = 200,
* there will be 400 write requests (200 on each thread).
*/
private static final int TOTAL_WRITES_PER_RUN = 4000;

/**
* Number of requests which are used to initialise/warmup the benchmark. The latency number of
* these runs are ignored from the final reported results.
*/
private static final int WARMUP_REQUEST_COUNT = 1;

/**
* Numbers of records in the sample table used in the benchmark. This is used in this benchmark to
* randomly choose a primary key and ensure that the reads are randomly distributed. This is done
* to ensure we don't end up reading/writing the same table record (leading to hot-spotting).
*/
private static final int TOTAL_RECORDS = 1000000;

@State(Scope.Thread)
@AuxCounters(org.openjdk.jmh.annotations.AuxCounters.Type.EVENTS)
public static class BenchmarkState {
Expand All @@ -115,12 +80,20 @@ public static class BenchmarkState {
private Spanner spanner;
private DatabaseClientImpl client;

@Param({"100"})
int minSessions;

@Param({"400"})
int maxSessions;

@Setup(Level.Iteration)
public void setup() throws Exception {
SpannerOptions options =
SpannerOptions.newBuilder()
.setSessionPoolOption(
SessionPoolOptions.newBuilder()
.setMinSessions(minSessions)
.setMaxSessions(maxSessions)
.setWaitForMinSessions(org.threeten.bp.Duration.ofSeconds(20))
.build())
.setHost(SERVER_URL)
Expand Down Expand Up @@ -217,7 +190,8 @@ private java.time.Duration executeQuery(final BenchmarkState server) {

try (ResultSet rs = server.client.singleUse().executeQuery(getRandomisedReadStatement())) {
while (rs.next()) {
int count = rs.getColumnCount();
assertEquals(1, rs.getColumnCount());
assertNotNull(rs.getValue(0));
}
}
return watch.elapsed();
Expand Down
Loading