Skip to content

Commit

Permalink
timeseries, implement BigDoubleBuffer and BigStringBuffer with testab…
Browse files Browse the repository at this point in the history 
…ility without bytebuffer

Signed-off-by: HARPER Jon <jon.harper87@gmail.com>
  • Loading branch information
@jonenst
jonenst committed Mar 9, 2023
1 parent 5b0cf6f commit e63d9e6
Show file tree
Hide file tree
Showing 5 changed files with 220 additions and 149 deletions.
49 changes: 37 additions & 12 deletions time-series/time-series-api/src/main/java/com/powsybl/timeseries/BigDoubleBuffer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import java.nio.DoubleBuffer;
import java.util.Objects;
import java.util.function.IntFunction;
import java.util.function.IntConsumer;

/**
* @author Jon Harper <jon.harper at rte-france.com>
Expand All @@ -23,8 +24,13 @@ public class BigDoubleBuffer {
private DoubleBuffer[] buffers;
private long size;

public BigDoubleBuffer(IntFunction<ByteBuffer> byteBufferAllocator, long size) {
Objects.requireNonNull(byteBufferAllocator);
//To remove if we ever get it from somewhere else
//package private for tests
@FunctionalInterface interface IntIntBiConsumer { public void accept(int a, int b); }

//using a lambda to test independently from java.nio.ByteBuffer
//package private for tests
static void withSizes(long size, IntConsumer bufferContainerInitializer, IntIntBiConsumer bufferInitializer) {
if (size < 0) {
throw new IllegalArgumentException("Invalid buffer size: " + size);
}
Expand All @@ -38,30 +44,49 @@ public BigDoubleBuffer(IntFunction<ByteBuffer> byteBufferAllocator, long size) {
if (size > 0 && lastBufferSizeBytes == 0) {
lastBufferSizeBytes = BUFFER_SIZE_BYTES;
}
buffers = new DoubleBuffer[bufferCount];
bufferContainerInitializer.accept(bufferCount);
for (int i = 0; i < bufferCount - 1; i++) {
buffers[i] = byteBufferAllocator.apply(BUFFER_SIZE_BYTES).asDoubleBuffer();
bufferInitializer.accept(i, BUFFER_SIZE_BYTES);
}
if (lastBufferSizeBytes > 0) {
buffers[bufferCount - 1] = byteBufferAllocator.apply(lastBufferSizeBytes).asDoubleBuffer();
bufferInitializer.accept(bufferCount - 1, lastBufferSizeBytes);
}
}

public BigDoubleBuffer(IntFunction<ByteBuffer> byteBufferAllocator, long size) {
Objects.requireNonNull(byteBufferAllocator);
withSizes(size,
bufferCount -> buffers = new DoubleBuffer[bufferCount],
(i, bufferSize) -> buffers[i] = byteBufferAllocator.apply(bufferSize).asDoubleBuffer()
);
this.size = size;
}

public void put(long index, double value) {
//To remove if we ever get it from somewhere else
//package private for tests
@FunctionalInterface interface IntIntToDoubleBiFunction { public double applyAsDouble(int a, int b); }

//using a lambda to test independently from java.nio.ByteBuffer
//package private for tests
static double withIndices(long index, IntIntToDoubleBiFunction indicesBiFunction) {
long computedBufferIndex = index >> BUFFER_SHIFT;
long computedSecondIndex = index & BUFFER_MASK;
int bufferIndex = (int) computedBufferIndex;
int secondIndex = (int) computedSecondIndex;
buffers[bufferIndex].put(secondIndex, value);
return indicesBiFunction.applyAsDouble(bufferIndex, secondIndex);
}

public void put(long index, double value) {
withIndices(index, (bufferIndex, secondIndex) -> {
buffers[bufferIndex].put(secondIndex, value);
return Double.NaN; // just to reuse the withIndices code
});
}

public double get(long index) {
long computedBufferIndex = index >> BUFFER_SHIFT;
long computedSecondIndex = index & BUFFER_MASK;
int bufferIndex = (int) computedBufferIndex;
int secondIndex = (int) computedSecondIndex;
return buffers[bufferIndex].get(secondIndex);
return withIndices(index, (bufferIndex, secondIndex) ->
buffers[bufferIndex].get(secondIndex)
);
}

public long capacity() {
Expand Down
49 changes: 37 additions & 12 deletions time-series/time-series-api/src/main/java/com/powsybl/timeseries/BigStringBuffer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.nio.ByteBuffer;
import java.util.Objects;
import java.util.function.IntFunction;
import java.util.function.IntConsumer;

/**
* @author Jon Harper <jon.harper at rte-france.com>
Expand All @@ -21,8 +22,13 @@ public class BigStringBuffer {
private CompactStringBuffer[] buffers;
private long size;

public BigStringBuffer(IntFunction<ByteBuffer> byteBufferAllocator, long size) {
Objects.requireNonNull(byteBufferAllocator);
//To remove if we ever get it from somewhere else
//package private for tests
@FunctionalInterface interface IntIntBiConsumer { public void accept(int a, int b); }

//using a lambda to test independently from java.nio.ByteBuffer
//package private for tests
static void withSizes(long size, IntConsumer bufferContainerInitializer, IntIntBiConsumer bufferInitializer) {
if (size < 0) {
throw new IllegalArgumentException("Invalid buffer size: " + size);
}
Expand All @@ -36,30 +42,49 @@ public BigStringBuffer(IntFunction<ByteBuffer> byteBufferAllocator, long size) {
if (size > 0 && lastBufferSizeInts == 0) {
lastBufferSizeInts = BUFFER_SIZE_INTS;
}
buffers = new CompactStringBuffer[bufferCount];
bufferContainerInitializer.accept(bufferCount);
for (int i = 0; i < bufferCount - 1; i++) {
buffers[i] = new CompactStringBuffer(byteBufferAllocator, BUFFER_SIZE_INTS);
bufferInitializer.accept(i, BUFFER_SIZE_INTS);
}
if (lastBufferSizeInts > 0) {
buffers[bufferCount - 1] = new CompactStringBuffer(byteBufferAllocator, lastBufferSizeInts);
bufferInitializer.accept(bufferCount - 1, lastBufferSizeInts);
}
}

public BigStringBuffer(IntFunction<ByteBuffer> byteBufferAllocator, long size) {
Objects.requireNonNull(byteBufferAllocator);
withSizes(size,
bufferCount -> buffers = new CompactStringBuffer[bufferCount],
(i, bufferSize) -> buffers[i] = new CompactStringBuffer(byteBufferAllocator, bufferSize)
);
this.size = size;
}

public void putString(long index, String value) {
//To remove if we ever get it from somewhere else
//package private for tests
@FunctionalInterface interface IntIntBiFunction { public String apply(int a, int b); }

//using a lambda to test independently from java.nio.ByteBuffer
//package private for tests
static String withIndices(long index, IntIntBiFunction indicesBiFunction) {
long computedBufferIndex = index >> BUFFER_SHIFT;
long computedSecondIndex = index & BUFFER_MASK;
int bufferIndex = (int) computedBufferIndex;
int secondIndex = (int) computedSecondIndex;
buffers[bufferIndex].putString(secondIndex, value);
return indicesBiFunction.apply(bufferIndex, secondIndex);
}

public void putString(long index, String value) {
withIndices(index, (bufferIndex, secondIndex) -> {
buffers[bufferIndex].putString(secondIndex, value);
return null; // just to reuse the withIndices code
});
}

public String getString(long index) {
long computedBufferIndex = index >> BUFFER_SHIFT;
long computedSecondIndex = index & BUFFER_MASK;
int bufferIndex = (int) computedBufferIndex;
int secondIndex = (int) computedSecondIndex;
return buffers[bufferIndex].getString(secondIndex);
return withIndices(index, (bufferIndex, secondIndex) ->
buffers[bufferIndex].getString(secondIndex)
);
}

public long capacity() {
Expand Down
57 changes: 0 additions & 57 deletions time-series/time-series-api/src/test/java/com/powsybl/timeseries/AbstractBigBufferTest.java
View file

This file was deleted.

107 changes: 73 additions & 34 deletions time-series/time-series-api/src/test/java/com/powsybl/timeseries/BigDoubleBufferTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,80 +6,119 @@
*/
package com.powsybl.timeseries;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;

import java.nio.ByteBuffer;
import java.util.function.IntConsumer;
import com.powsybl.timeseries.BigDoubleBuffer.IntIntBiConsumer;
import com.powsybl.timeseries.BigDoubleBuffer.IntIntToDoubleBiFunction;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.*;

/**
* @author Jon Harper <jon.harper at rte-france.com>
*/
class BigDoubleBufferTest extends AbstractBigBufferTest {
class BigDoubleBufferTest {

private static final int BUFFER_SIZE_DOUBLES = 1 << 27;

private void bufferTester(long size) {
BigDoubleBuffer buffer = new BigDoubleBuffer(this::testAllocator, size);
private int allocatorCount;

private ByteBuffer testDoubleAllocator(int capacity) {
try {
ByteBuffer bytebuffer = ByteBuffer.allocate(capacity);
allocatorCount++;
return bytebuffer;
} catch (Exception e) {
throw new RuntimeException("error in allocator test", e);
}
}

@BeforeEach
void before() {
allocatorCount = 0;
}

@Test
void testSimple() {
long size = 10L;
BigDoubleBuffer buffer = new BigDoubleBuffer(this::testDoubleAllocator, size);
assertEquals(1, allocatorCount);
//Simple writes at the begining
for (int i = 0; i < 10; i++) {
buffer.put(i, i);
}
//writes around first buffer change
if (size > BUFFER_SIZE_DOUBLES + 10) {
for (int i = BUFFER_SIZE_DOUBLES - 10; i < BUFFER_SIZE_DOUBLES + 10; i++) {
buffer.put(i, i);
}
}
//writes at the end
for (long i = size - 10; i < size; i++) {
buffer.put(i, i);
}

for (int i = 0; i < 10; i++) {
assertEquals(i, buffer.get(i), 0);
}
if (size > BUFFER_SIZE_DOUBLES + 10) {
for (int i = BUFFER_SIZE_DOUBLES - 10; i < BUFFER_SIZE_DOUBLES + 10; i++) {
assertEquals(i, buffer.get(i), 0);
}
}
for (long i = size - 10; i < size; i++) {
assertEquals(i, buffer.get(i), 0);
}
}

@Test
void testSimple() {
bufferTester(10);
assertEquals(1, channels.size());
void testWithIndices() {
IntIntToDoubleBiFunction indicesBiFunction = Mockito.mock(IntIntToDoubleBiFunction.class);
BigDoubleBuffer.withIndices(10, indicesBiFunction);
verify(indicesBiFunction).applyAsDouble(0, 10);
verifyNoMoreInteractions(indicesBiFunction);

//writes around first buffer change
BigDoubleBuffer.withIndices(BUFFER_SIZE_DOUBLES - 1, indicesBiFunction);
verify(indicesBiFunction).applyAsDouble(0, BUFFER_SIZE_DOUBLES - 1);
verifyNoMoreInteractions(indicesBiFunction);
BigDoubleBuffer.withIndices(BUFFER_SIZE_DOUBLES, indicesBiFunction);
verify(indicesBiFunction).applyAsDouble(1, 0);
verifyNoMoreInteractions(indicesBiFunction);

//writes around random buffer change
BigDoubleBuffer.withIndices(7 * BUFFER_SIZE_DOUBLES - 1, indicesBiFunction);
verify(indicesBiFunction).applyAsDouble(6, BUFFER_SIZE_DOUBLES - 1);
verifyNoMoreInteractions(indicesBiFunction);
BigDoubleBuffer.withIndices(7 * BUFFER_SIZE_DOUBLES, indicesBiFunction);
verify(indicesBiFunction).applyAsDouble(7, 0);
verifyNoMoreInteractions(indicesBiFunction);
}

void indicesTester(long size, int bufferCount) {
IntConsumer bufferContainerInitializer = Mockito.mock(IntConsumer.class);
IntIntBiConsumer bufferInitializer = Mockito.mock(IntIntBiConsumer.class);
BigDoubleBuffer.withSizes(size,
bufferContainerInitializer,
bufferInitializer
);
verify(bufferContainerInitializer).accept(bufferCount);
verifyNoMoreInteractions(bufferContainerInitializer);
for (int i = 0; i < bufferCount - 1; i++) {
verify(bufferInitializer).accept(i, BUFFER_SIZE_DOUBLES * Double.BYTES);
}
verify(bufferInitializer).accept(bufferCount - 1, (int) (size - (bufferCount - 1) * BUFFER_SIZE_DOUBLES) * Double.BYTES);
verifyNoMoreInteractions(bufferInitializer);
}

@Test
void testMultipleBuffers() {
bufferTester(200000000);
assertEquals(2, channels.size());
indicesTester(200000000, 2);
}

@Test
void testHuge() {
bufferTester(10000000000L);
assertEquals(75, channels.size());
indicesTester(10000000000L, 75);
}

@Test
void testSizeBufferMinus1() {
bufferTester(BUFFER_SIZE_DOUBLES - 1);
assertEquals(1, channels.size());
indicesTester(BUFFER_SIZE_DOUBLES - 1, 1);
}

@Test
void testSizeBufferExact() {
bufferTester(BUFFER_SIZE_DOUBLES);
assertEquals(1, channels.size());
indicesTester(BUFFER_SIZE_DOUBLES, 1);
}

@Test
void testSizeBufferPlus1() {
bufferTester(BUFFER_SIZE_DOUBLES + 1);
assertEquals(2, channels.size());
indicesTester(BUFFER_SIZE_DOUBLES + 1, 2);
}
}
Loading

0 comments on commit e63d9e6

Please sign in to comment.