From 1640f889b121844e3ff48a0370dd7df2d040ccd9 Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Fri, 15 Jul 2022 12:54:10 +0800 Subject: [PATCH 01/18] Add following utilities * `DataSize#add(long bytes)`, add by an amount of byte * `DataSize#subtract(long bytes)`, subtract by an amount of byte * `DataSize#bytes()`, a way to get bytes from DataSize, might overflow tho. * `DataSize#perSecond()`, a way to convert data size to data rate * `DataRate#doubleByteRate()`, a way to convert DataRate to bytes/second in double * `DataRate#longByteRate()`, a way to convert DataRate to bytes/second in long * `DataRate#dataSize()`, a way to convert DataRate to size per second * `DataRate.fromDouble()`, a way to convert a double measurement with byte/second as unit to DataRate * `DataRate.fromLong()`, a way to convert a long measurement with byte/second as unit to DataRate --- .../java/org/astraea/app/common/DataRate.java | 50 +++++++++++- .../java/org/astraea/app/common/DataSize.java | 40 +++++++++- .../org/astraea/app/common/DataRateTest.java | 42 ++++++++++ .../org/astraea/app/common/DataSizeTest.java | 76 +++++++++++++++++++ 4 files changed, 204 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/org/astraea/app/common/DataRate.java b/app/src/main/java/org/astraea/app/common/DataRate.java index e056427ecc..fa7981f4dc 100644 --- a/app/src/main/java/org/astraea/app/common/DataRate.java +++ b/app/src/main/java/org/astraea/app/common/DataRate.java @@ -89,7 +89,29 @@ public BigDecimal toBigDecimal(DataUnit dataUnit, Duration duration) { return this.totalBitTransmitted .measurement(dataUnit) .multiply(fromDurationToBigDecimalSafely(duration)) - .divide(durationInNanoSecond, MathContext.DECIMAL32); + .divide(durationInNanoSecond, MathContext.UNLIMITED); + } + + /** + * @return the data rate with bytes/second unit as a double value. If the value is beyond the + * range of double, it will become {@link Double#POSITIVE_INFINITY}. + */ + public double doubleByteRate() { + return toBigDecimal(DataUnit.Byte, ChronoUnit.SECONDS).doubleValue(); + } + + /** + * @return the data rate with bytes/second unit as a long value + * @throws ArithmeticException if the value overflowed. + */ + public long longByteRate() { + return toBigDecimal(DataUnit.Byte, ChronoUnit.SECONDS).toBigInteger().longValueExact(); + } + + /** @return the data rate per second as a {@link DataSize}. */ + public DataSize dataSize() { + var bitsPerSecond = toBigDecimal(DataUnit.Bit, ChronoUnit.SECONDS).toBigInteger(); + return new DataSize(bitsPerSecond); } /** @@ -180,6 +202,32 @@ public static double ofDouble(DataSize size, DataUnit unit, Duration time) { return ofBigDecimal(size, unit, time).doubleValue(); } + /** + * @param bytesPerSecond the double value that represent a data rate in bytes/second unit + * @return a {@link DataRate} converted from the given parameter. + */ + public static DataRate fromDouble(double bytesPerSecond) { + var bits = + BigDecimal.valueOf(bytesPerSecond) + .multiply(new BigDecimal(DataUnit.Byte.bits)) + .toBigInteger(); + var size = new DataSize(bits); + return new DataRate(size, Duration.ofSeconds(1)); + } + + /** + * @param bytesPerSecond the double value that represent a data rate in bytes/second unit + * @return a {@link DataRate} converted from the given parameter. + */ + public static DataRate fromLong(long bytesPerSecond) { + var bits = + BigDecimal.valueOf(bytesPerSecond) + .multiply(new BigDecimal(DataUnit.Byte.bits)) + .toBigInteger(); + var size = new DataSize(bits); + return new DataRate(size, Duration.ofSeconds(1)); + } + private static String chronoName(ChronoUnit chronoUnit) { return chronoUnit.name().toLowerCase(Locale.ROOT); } diff --git a/app/src/main/java/org/astraea/app/common/DataSize.java b/app/src/main/java/org/astraea/app/common/DataSize.java index 76889b11d5..e094a4e524 100644 --- a/app/src/main/java/org/astraea/app/common/DataSize.java +++ b/app/src/main/java/org/astraea/app/common/DataSize.java @@ -36,8 +36,28 @@ public class DataSize implements Comparable { this(BigInteger.valueOf(volume).multiply(dataUnit.bits)); } - DataSize(BigInteger bigInteger) { - this.bits = bigInteger; + public DataSize(BigInteger bits) { + this.bits = bits; + } + + /** + * Add data volume. + * + * @param bytes value to add + * @return a new {@link DataSize} that have applied the math operation. + */ + public DataSize add(long bytes) { + return add(bytes, DataUnit.Byte); + } + + /** + * Subtract data volume. + * + * @param bytes value to subtract + * @return a new {@link DataSize} that have applied the math operation. + */ + public DataSize subtract(long bytes) { + return subtract(bytes, DataUnit.Byte); } /** @@ -139,6 +159,15 @@ public BigInteger bits() { return bits; } + /** + * @return the current bytes in long primitive type. All the remaining bits that can't become a + * byte are discarded from the return value. + * @throws ArithmeticException if the value will not exactly fit into a long. + */ + public long bytes() { + return bits().divide(BigInteger.valueOf(8)).longValueExact(); + } + /** * The measurement value in term of specific data unit. * @@ -146,7 +175,7 @@ public BigInteger bits() { * @return a {@link BigDecimal} describe current data size in term of specific data unit. */ public BigDecimal measurement(DataUnit dataUnit) { - return new BigDecimal(this.bits).divide(new BigDecimal(dataUnit.bits), MathContext.DECIMAL32); + return new BigDecimal(this.bits).divide(new BigDecimal(dataUnit.bits), MathContext.UNLIMITED); } /** @@ -197,6 +226,11 @@ public DataRate dataRate(Duration timePassed) { return DataRate.of(this, timePassed); } + /** @return a {@link DataRate} based on current data size over one second. */ + public DataRate perSecond() { + return dataRate(ChronoUnit.SECONDS); + } + /** Return a string represent current size in given data unit. */ public String toString(DataUnit unit) { var divide = diff --git a/app/src/test/java/org/astraea/app/common/DataRateTest.java b/app/src/test/java/org/astraea/app/common/DataRateTest.java index 09bc026cfe..2d530eeddd 100644 --- a/app/src/test/java/org/astraea/app/common/DataRateTest.java +++ b/app/src/test/java/org/astraea/app/common/DataRateTest.java @@ -23,6 +23,8 @@ import java.time.Duration; import java.time.temporal.ChronoUnit; +import java.util.function.BiConsumer; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -100,4 +102,44 @@ void testDataRate( expectedIdealDataRate, sut.idealDataRate(ChronoUnit.SECONDS).doubleValue()); assertEquals(expectedDataUnit, sut.idealDataUnit(ChronoUnit.SECONDS)); } + + @Test + void testToDataSize() { + Assertions.assertEquals(DataUnit.Byte.of(1024), DataUnit.Byte.of(1024).perSecond().dataSize()); + Assertions.assertEquals(DataUnit.KiB.of(1024), DataUnit.KiB.of(1024).perSecond().dataSize()); + Assertions.assertEquals(DataUnit.EiB.of(24), DataUnit.EiB.of(24).perSecond().dataSize()); + } + + @Test + void testDoubleByteRate() { + BiConsumer assertDoubleEqual = + (a, b) -> { + Assertions.assertTrue( + Math.abs(a - b) < 1e-8, + "The value " + a + " and " + b + " should have no difference above 1e-8"); + }; + + assertDoubleEqual.accept(1024.0, DataUnit.Byte.of(1024).perSecond().doubleByteRate()); + assertDoubleEqual.accept(1024.0 * 1024, DataUnit.KiB.of(1024).perSecond().doubleByteRate()); + } + + @Test + void testLongByteRate() { + Assertions.assertEquals(1024L, DataUnit.Byte.of(1024).perSecond().longByteRate()); + Assertions.assertEquals(1024L * 1024, DataUnit.KiB.of(1024).perSecond().longByteRate()); + } + + @Test + void testFromLong() { + Assertions.assertEquals( + DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).longByteRate(), + DataRate.fromLong(1024).longByteRate()); + } + + @Test + void testFromDouble() { + Assertions.assertEquals( + DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).longByteRate(), + DataRate.fromDouble(1024).longByteRate()); + } } diff --git a/app/src/test/java/org/astraea/app/common/DataSizeTest.java b/app/src/test/java/org/astraea/app/common/DataSizeTest.java index 0b9f0fa826..fcb5e99665 100644 --- a/app/src/test/java/org/astraea/app/common/DataSizeTest.java +++ b/app/src/test/java/org/astraea/app/common/DataSizeTest.java @@ -30,6 +30,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.function.BiFunction; import java.util.function.Consumer; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -85,6 +86,29 @@ void typicalUsageOfDataUnit() { double double2 = DataRate.ofDouble(randomSize, DataUnit.Byte, ChronoUnit.SECONDS); double double3 = DataRate.ofDouble(randomSize, DataUnit.Byte, Duration.ofSeconds(1)); + // fast way to get bytes, be aware of exception caused by overflow. + long bytesInLong = randomSize.bytes(); + long bytes10Gib = DataUnit.Gib.of(10).bytes(); + + // fast way & normal way to get second rate + DataRate faster = DataUnit.Byte.of(1000).perSecond(); + DataRate normal = DataRate.of(1000, DataUnit.Byte, ChronoUnit.SECONDS); + + // data rate to other types + long dataRateLong = DataUnit.Byte.of(1000).perSecond().longByteRate(); + double dataRateDouble = DataUnit.Byte.of(1000).perSecond().doubleByteRate(); + DataSize dataRateSize = DataUnit.Byte.of(1000).perSecond().dataSize(); + + // fast way to get DataSize & DataRate from primitive type + DataSize primitive0 = DataUnit.Byte.of(1000); + DataSize primitive1 = DataUnit.Byte.of((long) 1000.0); + DataRate primitive2 = DataRate.fromDouble(123.0); + DataRate primitive3 = DataRate.fromLong(1024); + + // fast way to add/subtract data from primitive type + DataUnit.Byte.of(1000).subtract(500); + DataUnit.Byte.of(1024).add(1024); + // solve the above problem var dataVolume = DataUnit.YB.of(1); var dataVolumeOver1000Years = dataVolume.dataRate(ChronoUnit.MILLENNIA); @@ -315,4 +339,56 @@ void compare() { assertFalse(DataUnit.MB.of(0).greaterEqualTo(DataUnit.KB.of(1000))); assertFalse(DataUnit.MB.of(1).greaterThan(DataUnit.KB.of(1001))); } + + @ParameterizedTest + @CsvSource( + delimiterString = " ", + value = {"1000 KiB", "1234 Kib", "5566 GB", "42 Byte"}) + void perSecond(long measurement, DataUnit unit) { + var dataSize = unit.of(measurement); + var dataRate = dataSize.perSecond(); + + Assertions.assertEquals( + measurement, dataRate.toBigDecimal(unit, ChronoUnit.SECONDS).longValueExact()); + Assertions.assertEquals( + measurement * 60, dataRate.toBigDecimal(unit, ChronoUnit.MINUTES).longValueExact()); + } + + @ParameterizedTest + @CsvSource( + delimiterString = ",", + value = { + // measurement, unit, expected-value + " 1, KiB, 1024", + " 1000, GB, 1000000000000", + " 8, Bit, 1", + " 7, Bit, 0", + " 0, Bit, 0", + " 9, Bit, 1", + " 15, Bit, 1", + " 16, Bit, 2", + " 800, Bit, 100" + }) + void bytes(long measurement, DataUnit unit, long expected) { + var dataSize = unit.of(measurement); + + Assertions.assertEquals(expected, dataSize.bytes()); + + // overflow cases + Assertions.assertThrows(ArithmeticException.class, () -> DataUnit.PiB.of(8192).bytes()); + Assertions.assertDoesNotThrow(() -> DataUnit.PiB.of(8191).bytes()); + Assertions.assertDoesNotThrow(() -> DataUnit.Byte.of(Long.MAX_VALUE).bytes()); + } + + @Test + void addBytes() { + Assertions.assertEquals(1100, DataUnit.Byte.of(1000).add(100).bytes()); + Assertions.assertEquals(1124, DataUnit.KiB.of(1).add(100).bytes()); + } + + @Test + void subtractBytes() { + Assertions.assertEquals(900, DataUnit.Byte.of(1000).subtract(100).bytes()); + Assertions.assertEquals(924, DataUnit.KiB.of(1).subtract(100).bytes()); + } } From cfd3d090947160573cfb1da819af6f1c449fcbbb Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Fri, 15 Jul 2022 15:05:22 +0800 Subject: [PATCH 02/18] improve the precision of divide operation --- app/src/main/java/org/astraea/app/common/DataRate.java | 2 +- app/src/main/java/org/astraea/app/common/DataSize.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/org/astraea/app/common/DataRate.java b/app/src/main/java/org/astraea/app/common/DataRate.java index fa7981f4dc..e780a7c3ae 100644 --- a/app/src/main/java/org/astraea/app/common/DataRate.java +++ b/app/src/main/java/org/astraea/app/common/DataRate.java @@ -89,7 +89,7 @@ public BigDecimal toBigDecimal(DataUnit dataUnit, Duration duration) { return this.totalBitTransmitted .measurement(dataUnit) .multiply(fromDurationToBigDecimalSafely(duration)) - .divide(durationInNanoSecond, MathContext.UNLIMITED); + .divide(durationInNanoSecond, MathContext.DECIMAL128); } /** diff --git a/app/src/main/java/org/astraea/app/common/DataSize.java b/app/src/main/java/org/astraea/app/common/DataSize.java index e094a4e524..c71d7ba0cb 100644 --- a/app/src/main/java/org/astraea/app/common/DataSize.java +++ b/app/src/main/java/org/astraea/app/common/DataSize.java @@ -175,7 +175,7 @@ public long bytes() { * @return a {@link BigDecimal} describe current data size in term of specific data unit. */ public BigDecimal measurement(DataUnit dataUnit) { - return new BigDecimal(this.bits).divide(new BigDecimal(dataUnit.bits), MathContext.UNLIMITED); + return new BigDecimal(this.bits).divide(new BigDecimal(dataUnit.bits), MathContext.DECIMAL128); } /** From bfbc004094fa91e1867152c284e217f6c3448b62 Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Fri, 15 Jul 2022 17:06:10 +0800 Subject: [PATCH 03/18] remove `DataRate#longByteRate` --- .../main/java/org/astraea/app/common/DataRate.java | 10 +--------- .../java/org/astraea/app/common/DataRateTest.java | 12 ++++++------ .../java/org/astraea/app/common/DataSizeTest.java | 1 - 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/app/src/main/java/org/astraea/app/common/DataRate.java b/app/src/main/java/org/astraea/app/common/DataRate.java index e780a7c3ae..bd8344f6d4 100644 --- a/app/src/main/java/org/astraea/app/common/DataRate.java +++ b/app/src/main/java/org/astraea/app/common/DataRate.java @@ -94,20 +94,12 @@ public BigDecimal toBigDecimal(DataUnit dataUnit, Duration duration) { /** * @return the data rate with bytes/second unit as a double value. If the value is beyond the - * range of double, it will become {@link Double#POSITIVE_INFINITY}. + * range of double, {@link Double#POSITIVE_INFINITY} will be returned. */ public double doubleByteRate() { return toBigDecimal(DataUnit.Byte, ChronoUnit.SECONDS).doubleValue(); } - /** - * @return the data rate with bytes/second unit as a long value - * @throws ArithmeticException if the value overflowed. - */ - public long longByteRate() { - return toBigDecimal(DataUnit.Byte, ChronoUnit.SECONDS).toBigInteger().longValueExact(); - } - /** @return the data rate per second as a {@link DataSize}. */ public DataSize dataSize() { var bitsPerSecond = toBigDecimal(DataUnit.Bit, ChronoUnit.SECONDS).toBigInteger(); diff --git a/app/src/test/java/org/astraea/app/common/DataRateTest.java b/app/src/test/java/org/astraea/app/common/DataRateTest.java index 2d530eeddd..a867ec5109 100644 --- a/app/src/test/java/org/astraea/app/common/DataRateTest.java +++ b/app/src/test/java/org/astraea/app/common/DataRateTest.java @@ -125,21 +125,21 @@ void testDoubleByteRate() { @Test void testLongByteRate() { - Assertions.assertEquals(1024L, DataUnit.Byte.of(1024).perSecond().longByteRate()); - Assertions.assertEquals(1024L * 1024, DataUnit.KiB.of(1024).perSecond().longByteRate()); + Assertions.assertEquals(1024L, DataUnit.Byte.of(1024).perSecond().doubleByteRate()); + Assertions.assertEquals(1024L * 1024, DataUnit.KiB.of(1024).perSecond().doubleByteRate()); } @Test void testFromLong() { Assertions.assertEquals( - DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).longByteRate(), - DataRate.fromLong(1024).longByteRate()); + DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).doubleByteRate(), + DataRate.fromLong(1024).doubleByteRate()); } @Test void testFromDouble() { Assertions.assertEquals( - DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).longByteRate(), - DataRate.fromDouble(1024).longByteRate()); + DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).doubleByteRate(), + DataRate.fromDouble(1024).doubleByteRate()); } } diff --git a/app/src/test/java/org/astraea/app/common/DataSizeTest.java b/app/src/test/java/org/astraea/app/common/DataSizeTest.java index fcb5e99665..7ae26d026a 100644 --- a/app/src/test/java/org/astraea/app/common/DataSizeTest.java +++ b/app/src/test/java/org/astraea/app/common/DataSizeTest.java @@ -95,7 +95,6 @@ void typicalUsageOfDataUnit() { DataRate normal = DataRate.of(1000, DataUnit.Byte, ChronoUnit.SECONDS); // data rate to other types - long dataRateLong = DataUnit.Byte.of(1000).perSecond().longByteRate(); double dataRateDouble = DataUnit.Byte.of(1000).perSecond().doubleByteRate(); DataSize dataRateSize = DataUnit.Byte.of(1000).perSecond().dataSize(); From 6ec7a42ee32c6dd1c674dd786c1b7c2584eaf050 Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Fri, 15 Jul 2022 17:07:21 +0800 Subject: [PATCH 04/18] remove `DataRate#longByteRate` --- .../java/org/astraea/app/common/DataRate.java | 2 +- .../org/astraea/app/common/DataRateTest.java | 16 ++++++++-------- .../org/astraea/app/common/DataSizeTest.java | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/org/astraea/app/common/DataRate.java b/app/src/main/java/org/astraea/app/common/DataRate.java index bd8344f6d4..53c4628efb 100644 --- a/app/src/main/java/org/astraea/app/common/DataRate.java +++ b/app/src/main/java/org/astraea/app/common/DataRate.java @@ -96,7 +96,7 @@ public BigDecimal toBigDecimal(DataUnit dataUnit, Duration duration) { * @return the data rate with bytes/second unit as a double value. If the value is beyond the * range of double, {@link Double#POSITIVE_INFINITY} will be returned. */ - public double doubleByteRate() { + public double byteRate() { return toBigDecimal(DataUnit.Byte, ChronoUnit.SECONDS).doubleValue(); } diff --git a/app/src/test/java/org/astraea/app/common/DataRateTest.java b/app/src/test/java/org/astraea/app/common/DataRateTest.java index a867ec5109..211944a27a 100644 --- a/app/src/test/java/org/astraea/app/common/DataRateTest.java +++ b/app/src/test/java/org/astraea/app/common/DataRateTest.java @@ -119,27 +119,27 @@ void testDoubleByteRate() { "The value " + a + " and " + b + " should have no difference above 1e-8"); }; - assertDoubleEqual.accept(1024.0, DataUnit.Byte.of(1024).perSecond().doubleByteRate()); - assertDoubleEqual.accept(1024.0 * 1024, DataUnit.KiB.of(1024).perSecond().doubleByteRate()); + assertDoubleEqual.accept(1024.0, DataUnit.Byte.of(1024).perSecond().byteRate()); + assertDoubleEqual.accept(1024.0 * 1024, DataUnit.KiB.of(1024).perSecond().byteRate()); } @Test void testLongByteRate() { - Assertions.assertEquals(1024L, DataUnit.Byte.of(1024).perSecond().doubleByteRate()); - Assertions.assertEquals(1024L * 1024, DataUnit.KiB.of(1024).perSecond().doubleByteRate()); + Assertions.assertEquals(1024L, DataUnit.Byte.of(1024).perSecond().byteRate()); + Assertions.assertEquals(1024L * 1024, DataUnit.KiB.of(1024).perSecond().byteRate()); } @Test void testFromLong() { Assertions.assertEquals( - DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).doubleByteRate(), - DataRate.fromLong(1024).doubleByteRate()); + DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).byteRate(), + DataRate.fromLong(1024).byteRate()); } @Test void testFromDouble() { Assertions.assertEquals( - DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).doubleByteRate(), - DataRate.fromDouble(1024).doubleByteRate()); + DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).byteRate(), + DataRate.fromDouble(1024).byteRate()); } } diff --git a/app/src/test/java/org/astraea/app/common/DataSizeTest.java b/app/src/test/java/org/astraea/app/common/DataSizeTest.java index 7ae26d026a..e795151a49 100644 --- a/app/src/test/java/org/astraea/app/common/DataSizeTest.java +++ b/app/src/test/java/org/astraea/app/common/DataSizeTest.java @@ -95,7 +95,7 @@ void typicalUsageOfDataUnit() { DataRate normal = DataRate.of(1000, DataUnit.Byte, ChronoUnit.SECONDS); // data rate to other types - double dataRateDouble = DataUnit.Byte.of(1000).perSecond().doubleByteRate(); + double dataRateDouble = DataUnit.Byte.of(1000).perSecond().byteRate(); DataSize dataRateSize = DataUnit.Byte.of(1000).perSecond().dataSize(); // fast way to get DataSize & DataRate from primitive type From ce0e693854e28f1e1801f6ed4d5e9f8ecb433f3c Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Fri, 15 Jul 2022 17:10:11 +0800 Subject: [PATCH 05/18] rename --- app/src/main/java/org/astraea/app/common/DataRate.java | 4 ++-- app/src/test/java/org/astraea/app/common/DataRateTest.java | 4 ++-- app/src/test/java/org/astraea/app/common/DataSizeTest.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/org/astraea/app/common/DataRate.java b/app/src/main/java/org/astraea/app/common/DataRate.java index 53c4628efb..25196412c6 100644 --- a/app/src/main/java/org/astraea/app/common/DataRate.java +++ b/app/src/main/java/org/astraea/app/common/DataRate.java @@ -198,7 +198,7 @@ public static double ofDouble(DataSize size, DataUnit unit, Duration time) { * @param bytesPerSecond the double value that represent a data rate in bytes/second unit * @return a {@link DataRate} converted from the given parameter. */ - public static DataRate fromDouble(double bytesPerSecond) { + public static DataRate ofDouble(double bytesPerSecond) { var bits = BigDecimal.valueOf(bytesPerSecond) .multiply(new BigDecimal(DataUnit.Byte.bits)) @@ -211,7 +211,7 @@ public static DataRate fromDouble(double bytesPerSecond) { * @param bytesPerSecond the double value that represent a data rate in bytes/second unit * @return a {@link DataRate} converted from the given parameter. */ - public static DataRate fromLong(long bytesPerSecond) { + public static DataRate ofLong(long bytesPerSecond) { var bits = BigDecimal.valueOf(bytesPerSecond) .multiply(new BigDecimal(DataUnit.Byte.bits)) diff --git a/app/src/test/java/org/astraea/app/common/DataRateTest.java b/app/src/test/java/org/astraea/app/common/DataRateTest.java index 211944a27a..ab48f4d1da 100644 --- a/app/src/test/java/org/astraea/app/common/DataRateTest.java +++ b/app/src/test/java/org/astraea/app/common/DataRateTest.java @@ -133,13 +133,13 @@ void testLongByteRate() { void testFromLong() { Assertions.assertEquals( DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).byteRate(), - DataRate.fromLong(1024).byteRate()); + DataRate.ofLong(1024).byteRate()); } @Test void testFromDouble() { Assertions.assertEquals( DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).byteRate(), - DataRate.fromDouble(1024).byteRate()); + DataRate.ofDouble(1024).byteRate()); } } diff --git a/app/src/test/java/org/astraea/app/common/DataSizeTest.java b/app/src/test/java/org/astraea/app/common/DataSizeTest.java index e795151a49..612eac2004 100644 --- a/app/src/test/java/org/astraea/app/common/DataSizeTest.java +++ b/app/src/test/java/org/astraea/app/common/DataSizeTest.java @@ -101,8 +101,8 @@ void typicalUsageOfDataUnit() { // fast way to get DataSize & DataRate from primitive type DataSize primitive0 = DataUnit.Byte.of(1000); DataSize primitive1 = DataUnit.Byte.of((long) 1000.0); - DataRate primitive2 = DataRate.fromDouble(123.0); - DataRate primitive3 = DataRate.fromLong(1024); + DataRate primitive2 = DataRate.ofDouble(123.0); + DataRate primitive3 = DataRate.ofLong(1024); // fast way to add/subtract data from primitive type DataUnit.Byte.of(1000).subtract(500); From c8d073454851591c033a6fb5cadc5b2747afde40 Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Sat, 16 Jul 2022 01:13:41 +0800 Subject: [PATCH 06/18] add `DataSize.ZERO` --- .../main/java/org/astraea/app/common/DataSize.java | 1 + .../java/org/astraea/app/common/DataSizeTest.java | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/app/src/main/java/org/astraea/app/common/DataSize.java b/app/src/main/java/org/astraea/app/common/DataSize.java index c71d7ba0cb..6f001e5045 100644 --- a/app/src/main/java/org/astraea/app/common/DataSize.java +++ b/app/src/main/java/org/astraea/app/common/DataSize.java @@ -30,6 +30,7 @@ /** Data size class */ public class DataSize implements Comparable { + public static final DataSize ZERO = DataUnit.Byte.of(0); private final BigInteger bits; DataSize(long volume, DataUnit dataUnit) { diff --git a/app/src/test/java/org/astraea/app/common/DataSizeTest.java b/app/src/test/java/org/astraea/app/common/DataSizeTest.java index 612eac2004..698d935f9d 100644 --- a/app/src/test/java/org/astraea/app/common/DataSizeTest.java +++ b/app/src/test/java/org/astraea/app/common/DataSizeTest.java @@ -30,6 +30,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.stream.IntStream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -86,6 +87,11 @@ void typicalUsageOfDataUnit() { double double2 = DataRate.ofDouble(randomSize, DataUnit.Byte, ChronoUnit.SECONDS); double double3 = DataRate.ofDouble(randomSize, DataUnit.Byte, Duration.ofSeconds(1)); + // sum all data size + var sumAll = + IntStream.range(0, 100).mapToObj(DataUnit.Byte::of).reduce(DataSize.ZERO, DataSize::add); + Assertions.assertEquals(4950, sumAll.bytes()); + // fast way to get bytes, be aware of exception caused by overflow. long bytesInLong = randomSize.bytes(); long bytes10Gib = DataUnit.Gib.of(10).bytes(); @@ -390,4 +396,10 @@ void subtractBytes() { Assertions.assertEquals(900, DataUnit.Byte.of(1000).subtract(100).bytes()); Assertions.assertEquals(924, DataUnit.KiB.of(1).subtract(100).bytes()); } + + @Test + void zero() { + Assertions.assertEquals(DataUnit.Bit.of(0), DataSize.ZERO); + Assertions.assertEquals(0, DataSize.ZERO.bits().longValue()); + } } From 89ca9bef4d707211a4453709be69d88823d1138a Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Sat, 16 Jul 2022 22:47:00 +0800 Subject: [PATCH 07/18] make DataSize constructor package-private --- app/src/main/java/org/astraea/app/common/DataSize.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/org/astraea/app/common/DataSize.java b/app/src/main/java/org/astraea/app/common/DataSize.java index 6f001e5045..ad837918cc 100644 --- a/app/src/main/java/org/astraea/app/common/DataSize.java +++ b/app/src/main/java/org/astraea/app/common/DataSize.java @@ -37,7 +37,7 @@ public class DataSize implements Comparable { this(BigInteger.valueOf(volume).multiply(dataUnit.bits)); } - public DataSize(BigInteger bits) { + DataSize(BigInteger bits) { this.bits = bits; } From c38726c654282b24ed3f6930e185faeee25bc82a Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Sat, 16 Jul 2022 22:53:18 +0800 Subject: [PATCH 08/18] rename `DataRate#ofDouble` and `DataRate#ofLong` to `DataRate#of` --- app/src/main/java/org/astraea/app/common/DataRate.java | 4 ++-- app/src/test/java/org/astraea/app/common/DataRateTest.java | 4 ++-- app/src/test/java/org/astraea/app/common/DataSizeTest.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/org/astraea/app/common/DataRate.java b/app/src/main/java/org/astraea/app/common/DataRate.java index 25196412c6..3a010f47fe 100644 --- a/app/src/main/java/org/astraea/app/common/DataRate.java +++ b/app/src/main/java/org/astraea/app/common/DataRate.java @@ -198,7 +198,7 @@ public static double ofDouble(DataSize size, DataUnit unit, Duration time) { * @param bytesPerSecond the double value that represent a data rate in bytes/second unit * @return a {@link DataRate} converted from the given parameter. */ - public static DataRate ofDouble(double bytesPerSecond) { + public static DataRate of(double bytesPerSecond) { var bits = BigDecimal.valueOf(bytesPerSecond) .multiply(new BigDecimal(DataUnit.Byte.bits)) @@ -211,7 +211,7 @@ public static DataRate ofDouble(double bytesPerSecond) { * @param bytesPerSecond the double value that represent a data rate in bytes/second unit * @return a {@link DataRate} converted from the given parameter. */ - public static DataRate ofLong(long bytesPerSecond) { + public static DataRate of(long bytesPerSecond) { var bits = BigDecimal.valueOf(bytesPerSecond) .multiply(new BigDecimal(DataUnit.Byte.bits)) diff --git a/app/src/test/java/org/astraea/app/common/DataRateTest.java b/app/src/test/java/org/astraea/app/common/DataRateTest.java index ab48f4d1da..643d4f33fa 100644 --- a/app/src/test/java/org/astraea/app/common/DataRateTest.java +++ b/app/src/test/java/org/astraea/app/common/DataRateTest.java @@ -133,13 +133,13 @@ void testLongByteRate() { void testFromLong() { Assertions.assertEquals( DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).byteRate(), - DataRate.ofLong(1024).byteRate()); + DataRate.of(1024).byteRate()); } @Test void testFromDouble() { Assertions.assertEquals( DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).byteRate(), - DataRate.ofDouble(1024).byteRate()); + DataRate.of(1024.0).byteRate()); } } diff --git a/app/src/test/java/org/astraea/app/common/DataSizeTest.java b/app/src/test/java/org/astraea/app/common/DataSizeTest.java index 698d935f9d..3e26cdd2b9 100644 --- a/app/src/test/java/org/astraea/app/common/DataSizeTest.java +++ b/app/src/test/java/org/astraea/app/common/DataSizeTest.java @@ -107,8 +107,8 @@ void typicalUsageOfDataUnit() { // fast way to get DataSize & DataRate from primitive type DataSize primitive0 = DataUnit.Byte.of(1000); DataSize primitive1 = DataUnit.Byte.of((long) 1000.0); - DataRate primitive2 = DataRate.ofDouble(123.0); - DataRate primitive3 = DataRate.ofLong(1024); + DataRate primitive2 = DataRate.of(123.0); + DataRate primitive3 = DataRate.of(1024); // fast way to add/subtract data from primitive type DataUnit.Byte.of(1000).subtract(500); From b8b9edfec0aa137986f656ac0c11f68f53c5d70f Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Sun, 17 Jul 2022 02:04:46 +0800 Subject: [PATCH 09/18] add constructor at DataSize --- .../java/org/astraea/app/common/DataSize.java | 42 +++++++++ .../org/astraea/app/common/DataSizeTest.java | 89 ++++++++++++++----- 2 files changed, 107 insertions(+), 24 deletions(-) diff --git a/app/src/main/java/org/astraea/app/common/DataSize.java b/app/src/main/java/org/astraea/app/common/DataSize.java index ad837918cc..a495f413da 100644 --- a/app/src/main/java/org/astraea/app/common/DataSize.java +++ b/app/src/main/java/org/astraea/app/common/DataSize.java @@ -31,6 +31,43 @@ public class DataSize implements Comparable { public static final DataSize ZERO = DataUnit.Byte.of(0); + + public static final DataSizeSource Bit = DataUnit.Bit::of; + public static final DataSizeSource Kb = DataUnit.Kb::of; + public static final DataSizeSource Mb = DataUnit.Mb::of; + public static final DataSizeSource Gb = DataUnit.Gb::of; + public static final DataSizeSource Tb = DataUnit.Tb::of; + public static final DataSizeSource Pb = DataUnit.Pb::of; + public static final DataSizeSource Eb = DataUnit.Eb::of; + public static final DataSizeSource Yb = DataUnit.Yb::of; + public static final DataSizeSource Zb = DataUnit.Zb::of; + public static final DataSizeSource Kib = DataUnit.Kib::of; + public static final DataSizeSource Mib = DataUnit.Mib::of; + public static final DataSizeSource Gib = DataUnit.Gib::of; + public static final DataSizeSource Tib = DataUnit.Tib::of; + public static final DataSizeSource Pib = DataUnit.Pib::of; + public static final DataSizeSource Eib = DataUnit.Eib::of; + public static final DataSizeSource Yib = DataUnit.Yib::of; + public static final DataSizeSource Zib = DataUnit.Zib::of; + + public static final DataSizeSource Byte = DataUnit.Byte::of; + public static final DataSizeSource KB = DataUnit.KB::of; + public static final DataSizeSource MB = DataUnit.MB::of; + public static final DataSizeSource GB = DataUnit.GB::of; + public static final DataSizeSource TB = DataUnit.TB::of; + public static final DataSizeSource PB = DataUnit.PB::of; + public static final DataSizeSource EB = DataUnit.EB::of; + public static final DataSizeSource YB = DataUnit.YB::of; + public static final DataSizeSource ZB = DataUnit.ZB::of; + public static final DataSizeSource KiB = DataUnit.KiB::of; + public static final DataSizeSource MiB = DataUnit.MiB::of; + public static final DataSizeSource GiB = DataUnit.GiB::of; + public static final DataSizeSource TiB = DataUnit.TiB::of; + public static final DataSizeSource PiB = DataUnit.PiB::of; + public static final DataSizeSource EiB = DataUnit.EiB::of; + public static final DataSizeSource YiB = DataUnit.YiB::of; + public static final DataSizeSource ZiB = DataUnit.ZiB::of; + private final BigInteger bits; DataSize(long volume, DataUnit dataUnit) { @@ -265,6 +302,11 @@ public int hashCode() { return Objects.hash(bits); } + @FunctionalInterface + public interface DataSizeSource { + DataSize of(long measurement); + } + public static class Field extends org.astraea.app.argument.Field { // Parse number and DataUnit private static final Pattern DATA_SIZE_PATTERN = diff --git a/app/src/test/java/org/astraea/app/common/DataSizeTest.java b/app/src/test/java/org/astraea/app/common/DataSizeTest.java index 3e26cdd2b9..38c608c48d 100644 --- a/app/src/test/java/org/astraea/app/common/DataSizeTest.java +++ b/app/src/test/java/org/astraea/app/common/DataSizeTest.java @@ -28,6 +28,7 @@ import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.concurrent.ThreadLocalRandom; +import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.stream.IntStream; @@ -42,39 +43,39 @@ class DataSizeTest { @Test void typicalUsageOfDataUnit() { // 500KB - DataUnit.KB.of(500); + DataSize.KB.of(500); // 500MB + 500MB - DataUnit.MB.of(500).add(500, DataUnit.MB); + DataSize.MB.of(500).add(500, DataUnit.MB); // 500GB - 500GB - DataUnit.GB.of(500).subtract(500, DataUnit.GB); + DataSize.GB.of(500).subtract(500, DataUnit.GB); // 500 TB / 10 - DataUnit.TB.of(500).divide(10); + DataSize.TB.of(500).divide(10); // 500 PB * 10 - DataUnit.PB.of(500).multiply(10); + DataSize.PB.of(500).multiply(10); // the data rate of sending 1 EB over 1 second - DataUnit.EB.of(1).dataRate(ChronoUnit.SECONDS); + DataSize.EB.of(1).dataRate(ChronoUnit.SECONDS); // the data rate of sending 1 EB over 1 millisecond - DataUnit.EB.of(1).dataRate(ChronoUnit.MILLIS); + DataSize.EB.of(1).dataRate(ChronoUnit.MILLIS); // the data rate of sending 1 EB over 1 century (100 years) - DataUnit.EB.of(1).dataRate(ChronoUnit.CENTURIES); + DataSize.EB.of(1).dataRate(ChronoUnit.CENTURIES); // the data rate of sending 1 EB over 59 seconds - DataUnit.EB.of(1).dataRate(Duration.ofSeconds(59)); + DataSize.EB.of(1).dataRate(Duration.ofSeconds(59)); // data rate string of sending 1 EB over 1 second in the most human friendly data unit & second // time unit - DataUnit.EB.of(1).dataRate(ChronoUnit.SECONDS).toString(); + DataSize.EB.of(1).dataRate(ChronoUnit.SECONDS).toString(); // data rate string of sending 1 EB over 1 second in the most human friendly data unit & 1 hour // time unit - DataUnit.EB.of(1).dataRate(ChronoUnit.SECONDS).toString(ChronoUnit.HOURS); + DataSize.EB.of(1).dataRate(ChronoUnit.SECONDS).toString(ChronoUnit.HOURS); // data rate string of sending 1 EB over 1 second in ZB unit and hour time unit - DataUnit.EB.of(1).dataRate(ChronoUnit.SECONDS).toString(DataUnit.ZB, ChronoUnit.HOURS); + DataSize.EB.of(1).dataRate(ChronoUnit.SECONDS).toString(DataUnit.ZB, ChronoUnit.HOURS); // someone wondering that if we send 1 YB of data over 1000 years, how much data we sent per // second. - DataUnit.YB.of(1).dataRate(ChronoUnit.MILLENNIA).toString(ChronoUnit.SECONDS); + DataSize.YB.of(1).dataRate(ChronoUnit.MILLENNIA).toString(ChronoUnit.SECONDS); // faster convert between DataRate and others. - var randomSize = DataUnit.Byte.of(ThreadLocalRandom.current().nextLong()); + var randomSize = DataSize.Byte.of(ThreadLocalRandom.current().nextLong()); BigDecimal bigDecimal0 = DataRate.ofBigDecimal(1000, DataUnit.Byte, ChronoUnit.SECONDS); BigDecimal bigDecimal1 = DataRate.ofBigDecimal(1000, DataUnit.Byte, Duration.ofSeconds(1)); @@ -89,33 +90,33 @@ void typicalUsageOfDataUnit() { // sum all data size var sumAll = - IntStream.range(0, 100).mapToObj(DataUnit.Byte::of).reduce(DataSize.ZERO, DataSize::add); + IntStream.range(0, 100).mapToObj(DataSize.Byte::of).reduce(DataSize.ZERO, DataSize::add); Assertions.assertEquals(4950, sumAll.bytes()); // fast way to get bytes, be aware of exception caused by overflow. long bytesInLong = randomSize.bytes(); - long bytes10Gib = DataUnit.Gib.of(10).bytes(); + long bytes10Gib = DataSize.Gib.of(10).bytes(); // fast way & normal way to get second rate - DataRate faster = DataUnit.Byte.of(1000).perSecond(); + DataRate faster = DataSize.Byte.of(1000).perSecond(); DataRate normal = DataRate.of(1000, DataUnit.Byte, ChronoUnit.SECONDS); // data rate to other types - double dataRateDouble = DataUnit.Byte.of(1000).perSecond().byteRate(); - DataSize dataRateSize = DataUnit.Byte.of(1000).perSecond().dataSize(); + double dataRateDouble = DataSize.Byte.of(1000).perSecond().byteRate(); + DataSize dataRateSize = DataSize.Byte.of(1000).perSecond().dataSize(); // fast way to get DataSize & DataRate from primitive type - DataSize primitive0 = DataUnit.Byte.of(1000); - DataSize primitive1 = DataUnit.Byte.of((long) 1000.0); + DataSize primitive0 = DataSize.Byte.of(1000); + DataSize primitive1 = DataSize.Byte.of((long) 1000.0); DataRate primitive2 = DataRate.of(123.0); DataRate primitive3 = DataRate.of(1024); // fast way to add/subtract data from primitive type - DataUnit.Byte.of(1000).subtract(500); - DataUnit.Byte.of(1024).add(1024); + DataSize.Byte.of(1000).subtract(500); + DataSize.Byte.of(1024).add(1024); // solve the above problem - var dataVolume = DataUnit.YB.of(1); + var dataVolume = DataSize.YB.of(1); var dataVolumeOver1000Years = dataVolume.dataRate(ChronoUnit.MILLENNIA); Consumer tellMeTheAnswerIn = (ChronoUnit chronoUnit) -> @@ -196,6 +197,46 @@ void of(String unitName, String expectedBits) { assertEquals(new BigInteger(expectedBits), DataUnit.valueOf(unitName).of(1).bits()); } + @Test + void dataSizeOf() { + BiConsumer test = Assertions::assertEquals; + + test.accept(new BigInteger("1"), DataSize.Bit.of(1).bits()); + test.accept(new BigInteger("1000"), DataSize.Kb.of(1).bits()); + test.accept(new BigInteger("1000000"), DataSize.Mb.of(1).bits()); + test.accept(new BigInteger("1000000000"), DataSize.Gb.of(1).bits()); + test.accept(new BigInteger("1000000000000"), DataSize.Tb.of(1).bits()); + test.accept(new BigInteger("1000000000000000"), DataSize.Pb.of(1).bits()); + test.accept(new BigInteger("1000000000000000000"), DataSize.Eb.of(1).bits()); + test.accept(new BigInteger("1000000000000000000000"), DataSize.Zb.of(1).bits()); + test.accept(new BigInteger("1000000000000000000000000"), DataSize.Yb.of(1).bits()); + test.accept(new BigInteger("1024"), DataSize.Kib.of(1).bits()); + test.accept(new BigInteger("1048576"), DataSize.Mib.of(1).bits()); + test.accept(new BigInteger("1073741824"), DataSize.Gib.of(1).bits()); + test.accept(new BigInteger("1099511627776"), DataSize.Tib.of(1).bits()); + test.accept(new BigInteger("1125899906842624"), DataSize.Pib.of(1).bits()); + test.accept(new BigInteger("1152921504606846976"), DataSize.Eib.of(1).bits()); + test.accept(new BigInteger("1180591620717411303424"), DataSize.Zib.of(1).bits()); + test.accept(new BigInteger("1208925819614629174706176"), DataSize.Yib.of(1).bits()); + test.accept(new BigInteger("8"), DataSize.Byte.of(1).bits()); + test.accept(new BigInteger("8000"), DataSize.KB.of(1).bits()); + test.accept(new BigInteger("8000000"), DataSize.MB.of(1).bits()); + test.accept(new BigInteger("8000000000"), DataSize.GB.of(1).bits()); + test.accept(new BigInteger("8000000000000"), DataSize.TB.of(1).bits()); + test.accept(new BigInteger("8000000000000000"), DataSize.PB.of(1).bits()); + test.accept(new BigInteger("8000000000000000000"), DataSize.EB.of(1).bits()); + test.accept(new BigInteger("8000000000000000000000"), DataSize.ZB.of(1).bits()); + test.accept(new BigInteger("8000000000000000000000000"), DataSize.YB.of(1).bits()); + test.accept(new BigInteger("8192"), DataSize.KiB.of(1).bits()); + test.accept(new BigInteger("8388608"), DataSize.MiB.of(1).bits()); + test.accept(new BigInteger("8589934592"), DataSize.GiB.of(1).bits()); + test.accept(new BigInteger("8796093022208"), DataSize.TiB.of(1).bits()); + test.accept(new BigInteger("9007199254740992"), DataSize.PiB.of(1).bits()); + test.accept(new BigInteger("9223372036854775808"), DataSize.EiB.of(1).bits()); + test.accept(new BigInteger("9444732965739290427392"), DataSize.ZiB.of(1).bits()); + test.accept(new BigInteger("9671406556917033397649408"), DataSize.YiB.of(1).bits()); + } + @Test void parseDataSize() { var converter = new DataSize.Field(); From 9cee4044e45c4f6a5965973236dbfe94ced85bf6 Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Sun, 17 Jul 2022 14:19:24 +0800 Subject: [PATCH 10/18] WIP --- .../app/admin/ReplicaSyncingMonitor.java | 6 ++--- .../org/astraea/app/admin/TopicExplorer.java | 11 ++++---- .../java/org/astraea/app/common/DataUnit.java | 2 +- .../org/astraea/app/performance/Manager.java | 4 +-- .../astraea/app/performance/Performance.java | 4 +-- .../astraea/app/performance/ReportFormat.java | 10 +++---- .../org/astraea/app/performance/Tracker.java | 6 ++--- .../app/admin/ReplicaSyncingMonitorTest.java | 7 ++--- .../balancer/executor/RebalanceAdminTest.java | 4 +-- .../app/performance/DataSupplierTest.java | 26 +++++++++---------- 10 files changed, 40 insertions(+), 40 deletions(-) diff --git a/app/src/main/java/org/astraea/app/admin/ReplicaSyncingMonitor.java b/app/src/main/java/org/astraea/app/admin/ReplicaSyncingMonitor.java index 5a7fd2dd7e..20685c8f99 100644 --- a/app/src/main/java/org/astraea/app/admin/ReplicaSyncingMonitor.java +++ b/app/src/main/java/org/astraea/app/admin/ReplicaSyncingMonitor.java @@ -120,11 +120,11 @@ static void execute(final Admin topicAdmin, final Argument argument) { Map.entry( entry.getValue(), new ProgressInfo( - DataUnit.Byte.of(leaderReplica.size()), - DataUnit.Byte.of( + DataSize.Byte.of(leaderReplica.size()), + DataSize.Byte.of( previousCheckedSize.getOrDefault( entry.getKey(), entry.getValue().size())), - DataUnit.Byte.of(entry.getValue().size()), + DataSize.Byte.of(entry.getValue().size()), argument.interval))) .map( entry -> diff --git a/app/src/main/java/org/astraea/app/admin/TopicExplorer.java b/app/src/main/java/org/astraea/app/admin/TopicExplorer.java index bf9d09278d..529f679af5 100644 --- a/app/src/main/java/org/astraea/app/admin/TopicExplorer.java +++ b/app/src/main/java/org/astraea/app/admin/TopicExplorer.java @@ -36,7 +36,6 @@ import org.astraea.app.argument.NonEmptyStringField; import org.astraea.app.argument.StringSetField; import org.astraea.app.common.DataSize; -import org.astraea.app.common.DataUnit; public class TopicExplorer { @@ -214,14 +213,14 @@ private void printStatistics(String topic, List partitionInfos) { () -> { treePrintln( "Topic Size: %s", - DataUnit.Byte.of( + DataSize.Byte.of( partitionInfos.stream() .flatMapToLong(x -> x.replicas.stream().mapToLong(Replica::size)) .sum())); treePrintln("Partition Count: %d", partitionInfos.size()); treePrintln( "Partition Size Average: %s", - DataUnit.Byte.of( + DataSize.Byte.of( (long) partitionInfos.stream() .mapToLong(x -> x.replicas.stream().mapToLong(Replica::size).sum()) @@ -232,7 +231,7 @@ private void printStatistics(String topic, List partitionInfos) { partitionInfos.stream().mapToInt(x -> x.replicas.size()).sum()); treePrintln( "Replica Size Average: %s", - DataUnit.Byte.of( + DataSize.Byte.of( (long) partitionInfos.stream() .flatMapToLong(x -> x.replicas.stream().mapToLong(Replica::size)) @@ -351,7 +350,7 @@ private void printPartitionReplica(List partitionInfos) { treePrintln( "Partition \"%d\" (size: %s) (offset range: [%d, %d])", partitionInfo.topicPartition.partition(), - DataUnit.Byte.of( + DataSize.Byte.of( partitionInfo.replicas.stream().mapToLong(Replica::size).sum()), partitionInfo.earliestOffset, partitionInfo.latestOffset); @@ -446,7 +445,7 @@ static String dataSizeString(DataSize dataSize) { } static String size(long bytes) { - var dataSize = DataUnit.Byte.of(bytes); + var dataSize = DataSize.Byte.of(bytes); return "(size=" + dataSizeString(dataSize) + ")"; } diff --git a/app/src/main/java/org/astraea/app/common/DataUnit.java b/app/src/main/java/org/astraea/app/common/DataUnit.java index 380ffcfc36..31a70e3404 100644 --- a/app/src/main/java/org/astraea/app/common/DataUnit.java +++ b/app/src/main/java/org/astraea/app/common/DataUnit.java @@ -105,7 +105,7 @@ public enum DataUnit { * @param measurement the data size measurement. * @return a size object of given measurement under specific data unit. */ - public DataSize of(long measurement) { + DataSize of(long measurement) { return new DataSize(measurement, this); } diff --git a/app/src/main/java/org/astraea/app/performance/Manager.java b/app/src/main/java/org/astraea/app/performance/Manager.java index d73e3127f2..98775bd074 100644 --- a/app/src/main/java/org/astraea/app/performance/Manager.java +++ b/app/src/main/java/org/astraea/app/performance/Manager.java @@ -18,7 +18,7 @@ import java.util.List; import java.util.function.Supplier; -import org.astraea.app.common.DataUnit; +import org.astraea.app.common.DataSize; /** * Thread safe This class is used for managing the start/end of the producer/consumer threads. @@ -47,7 +47,7 @@ public class Manager { */ public Manager( Performance.Argument argument, List producerMetrics, List consumerMetrics) { - if (argument.recordSize.greaterThan(DataUnit.Byte.of(Integer.MAX_VALUE))) + if (argument.recordSize.greaterThan(DataSize.Byte.of(Integer.MAX_VALUE))) throw new IllegalArgumentException( "Record size should be smaller than or equal to 2147483648 (Integer.MAX_VALUE) bytes"); this.producerMetrics = producerMetrics; diff --git a/app/src/main/java/org/astraea/app/performance/Performance.java b/app/src/main/java/org/astraea/app/performance/Performance.java index 7bdad597c5..6c36483347 100644 --- a/app/src/main/java/org/astraea/app/performance/Performance.java +++ b/app/src/main/java/org/astraea/app/performance/Performance.java @@ -314,7 +314,7 @@ public static class Argument extends org.astraea.app.argument.Argument { names = {"--record.size"}, description = "DataSize: size of each record. e.g. \"500KiB\"", converter = DataSize.Field.class) - DataSize recordSize = DataUnit.KiB.of(1); + DataSize recordSize = DataSize.KiB.of(1); @Parameter( names = {"--partitioner"}, @@ -365,7 +365,7 @@ Isolation isolation() { names = {"--throughput"}, description = "dataSize: size output per second. e.g. \"500KiB\"", converter = DataSize.Field.class) - DataSize throughput = DataUnit.GiB.of(500); + DataSize throughput = DataSize.GiB.of(500); @Parameter( names = {"--report.path"}, diff --git a/app/src/main/java/org/astraea/app/performance/ReportFormat.java b/app/src/main/java/org/astraea/app/performance/ReportFormat.java index 8a792d9cd2..f59a294450 100644 --- a/app/src/main/java/org/astraea/app/performance/ReportFormat.java +++ b/app/src/main/java/org/astraea/app/performance/ReportFormat.java @@ -28,7 +28,7 @@ import java.util.Date; import java.util.function.Supplier; import java.util.stream.IntStream; -import org.astraea.app.common.DataUnit; +import org.astraea.app.common.DataSize; import org.astraea.app.common.Utils; import org.astraea.app.concurrent.Executor; import org.astraea.app.concurrent.State; @@ -159,16 +159,16 @@ private static boolean logToCSV( + "s"); writer.write( String.format(",%.2f%% / %.2f%%", result.consumerPercentage, result.producerPercentage)); - writer.write("," + DataUnit.Byte.of(result.producerResult.totalCurrentBytes())); - writer.write("," + DataUnit.Byte.of(result.consumerResult.totalCurrentBytes())); + writer.write("," + DataSize.Byte.of(result.producerResult.totalCurrentBytes())); + writer.write("," + DataSize.Byte.of(result.consumerResult.totalCurrentBytes())); writer.write("," + result.producerResult.maxLatency + "," + result.producerResult.minLatency); writer.write("," + result.consumerResult.maxLatency + "," + result.consumerResult.minLatency); for (int i = 0; i < result.producerResult.bytes.size(); ++i) { - writer.write("," + DataUnit.Byte.of(result.producerResult.currentBytes.get(i))); + writer.write("," + DataSize.Byte.of(result.producerResult.currentBytes.get(i))); writer.write("," + result.producerResult.averageLatencies.get(i)); } for (int i = 0; i < result.consumerResult.bytes.size(); ++i) { - writer.write("," + DataUnit.Byte.of(result.consumerResult.currentBytes.get(i))); + writer.write("," + DataSize.Byte.of(result.consumerResult.currentBytes.get(i))); writer.write("," + result.consumerResult.averageLatencies.get(i)); } writer.newLine(); diff --git a/app/src/main/java/org/astraea/app/performance/Tracker.java b/app/src/main/java/org/astraea/app/performance/Tracker.java index c04d868762..6f0fee2bbb 100644 --- a/app/src/main/java/org/astraea/app/performance/Tracker.java +++ b/app/src/main/java/org/astraea/app/performance/Tracker.java @@ -21,7 +21,7 @@ import java.util.Collections; import java.util.List; import java.util.function.Supplier; -import org.astraea.app.common.DataUnit; +import org.astraea.app.common.DataSize; import org.astraea.app.concurrent.Executor; import org.astraea.app.concurrent.State; @@ -89,7 +89,7 @@ private boolean logProducers(Result result) { System.out.printf("producers completion rate: %.2f%%%n", percentage); System.out.printf(" average throughput: %.3f MB/second%n", result.averageBytes(duration)); System.out.printf( - " current throughput: %s/second%n", DataUnit.Byte.of(result.totalCurrentBytes())); + " current throughput: %s/second%n", DataSize.Byte.of(result.totalCurrentBytes())); System.out.println(" publish max latency: " + result.maxLatency + " ms"); System.out.println(" publish mim latency: " + result.minLatency + " ms"); for (int i = 0; i < result.bytes.size(); ++i) { @@ -113,7 +113,7 @@ private boolean logConsumers(Result result) { System.out.printf("consumer completion rate: %.2f%%%n", percentage); System.out.printf(" average throughput: %.3f MB/second%n", result.averageBytes(duration)); System.out.printf( - " current throughput: %s/second%n", DataUnit.Byte.of(result.totalCurrentBytes())); + " current throughput: %s/second%n", DataSize.Byte.of(result.totalCurrentBytes())); System.out.println(" end-to-end max latency: " + result.maxLatency + " ms"); System.out.println(" end-to-end mim latency: " + result.minLatency + " ms"); for (int i = 0; i < result.bytes.size(); ++i) { diff --git a/app/src/test/java/org/astraea/app/admin/ReplicaSyncingMonitorTest.java b/app/src/test/java/org/astraea/app/admin/ReplicaSyncingMonitorTest.java index ce2bfbfaca..94da5d372c 100644 --- a/app/src/test/java/org/astraea/app/admin/ReplicaSyncingMonitorTest.java +++ b/app/src/test/java/org/astraea/app/admin/ReplicaSyncingMonitorTest.java @@ -45,6 +45,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import org.astraea.app.argument.Argument; +import org.astraea.app.common.DataSize; import org.astraea.app.common.DataUnit; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -402,9 +403,9 @@ void dataRate( // act var progress = new ReplicaSyncingMonitor.ProgressInfo( - DataUnit.Byte.of(leaderSize), - DataUnit.Byte.of(previousSize), - DataUnit.Byte.of(currentSize), + DataSize.Byte.of(leaderSize), + DataSize.Byte.of(previousSize), + DataSize.Byte.of(currentSize), Duration.ofMillis(interval)); // assert diff --git a/app/src/test/java/org/astraea/app/balancer/executor/RebalanceAdminTest.java b/app/src/test/java/org/astraea/app/balancer/executor/RebalanceAdminTest.java index 0d33a8c7a1..e38eebf49a 100644 --- a/app/src/test/java/org/astraea/app/balancer/executor/RebalanceAdminTest.java +++ b/app/src/test/java/org/astraea/app/balancer/executor/RebalanceAdminTest.java @@ -93,7 +93,7 @@ void alterReplicaPlacementByDirectory() { var rebalanceAdmin = prepareRebalanceAdmin(admin); // decrease the debouncing time so the test has higher chance to fail RebalanceAdminImpl.changeRetrialTime(Duration.ofMillis(150)); - prepareData(topic, 0, DataUnit.MiB.of(256)); + prepareData(topic, 0, DataSize.MiB.of(256)); Supplier replicaNow = () -> admin.replicas(Set.of(topic)).get(topicPartition).get(0); var originalReplica = replicaNow.get(); var nextDir = @@ -148,7 +148,7 @@ void checkLogSynced() throws InterruptedException, ExecutionException { .findAny() .orElseThrow(); RebalanceAdminImpl.changeRetrialTime(Duration.ofMillis(150)); - prepareData(topic, 0, DataUnit.MiB.of(32)); + prepareData(topic, 0, DataSize.MiB.of(32)); // let two brokers join the replica list admin.migrator().partition(topic, 0).moveTo(List.of(0, 1, 2)); // let the existing replica change its directory diff --git a/app/src/test/java/org/astraea/app/performance/DataSupplierTest.java b/app/src/test/java/org/astraea/app/performance/DataSupplierTest.java index 3f5e51937f..6706a6d39c 100644 --- a/app/src/test/java/org/astraea/app/performance/DataSupplierTest.java +++ b/app/src/test/java/org/astraea/app/performance/DataSupplierTest.java @@ -17,7 +17,7 @@ package org.astraea.app.performance; import java.time.Duration; -import org.astraea.app.common.DataUnit; +import org.astraea.app.common.DataSize; import org.astraea.app.common.Utils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -30,9 +30,9 @@ void testDuration() { DataSupplier.of( ExeTime.of("2s"), DistributionType.FIXED.create(10), - DataUnit.KiB.of(100), + DataSize.KiB.of(100), DistributionType.FIXED.create(10), - DataUnit.KiB.of(100)); + DataSize.KiB.of(100)); Assertions.assertTrue(dataSupplier.get().hasData()); Utils.sleep(Duration.ofSeconds(3)); Assertions.assertFalse(dataSupplier.get().hasData()); @@ -44,9 +44,9 @@ void testRecordLimit() { DataSupplier.of( ExeTime.of("2records"), DistributionType.FIXED.create(10), - DataUnit.KiB.of(100), + DataSize.KiB.of(100), DistributionType.FIXED.create(10), - DataUnit.KiB.of(100)); + DataSize.KiB.of(100)); Assertions.assertTrue(dataSupplier.get().hasData()); Assertions.assertTrue(dataSupplier.get().hasData()); Assertions.assertFalse(dataSupplier.get().hasData()); @@ -58,9 +58,9 @@ void testKeySize() { DataSupplier.of( ExeTime.of("10s"), DistributionType.FIXED.create(9), - DataUnit.KiB.of(100), + DataSize.KiB.of(100), DistributionType.FIXED.create(10), - DataUnit.KiB.of(100)); + DataSize.KiB.of(100)); var data = dataSupplier.get(); Assertions.assertTrue(data.hasData()); // key content is fixed to "9", so the size is 1 byte @@ -73,9 +73,9 @@ void testFixedValueSize() { DataSupplier.of( ExeTime.of("10s"), DistributionType.FIXED.create(10), - DataUnit.KiB.of(100), + DataSize.KiB.of(100), DistributionType.FIXED.create(0), - DataUnit.KiB.of(100)); + DataSize.KiB.of(100)); var data = dataSupplier.get(); Assertions.assertTrue(data.hasData()); // initial value size is 100KB and the distributed is fixed to zero, so the final size is 102400 @@ -88,9 +88,9 @@ void testDistributedValueSize() { DataSupplier.of( ExeTime.of("10s"), DistributionType.FIXED.create(10), - DataUnit.KiB.of(100), + DataSize.KiB.of(100), DistributionType.FIXED.create(10), - DataUnit.KiB.of(100)); + DataSize.KiB.of(100)); var data = dataSupplier.get(); Assertions.assertTrue(data.hasData()); // initial value size is 100KB and the distributed is fixed to 10, so the final size is between @@ -104,9 +104,9 @@ void testThrottle() { DataSupplier.of( ExeTime.of("10s"), DistributionType.FIXED.create(10), - DataUnit.KiB.of(100), + DataSize.KiB.of(100), DistributionType.FIXED.create(0), - DataUnit.KiB.of(150)); + DataSize.KiB.of(150)); // total: 100KB, limit: 150KB -> no throttle Assertions.assertTrue(dataSupplier.get().hasData()); // total: 200KB, limit: 150KB -> will throttle next data From 57d813242cca7f151bf8ee3be65dc2a5f38f4c3a Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Mon, 18 Jul 2022 01:24:21 +0800 Subject: [PATCH 11/18] new way to create DataRate --- .../java/org/astraea/app/common/DataRate.java | 74 +++++++++++++++++++ .../org/astraea/app/common/DataRateTest.java | 65 ++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/app/src/main/java/org/astraea/app/common/DataRate.java b/app/src/main/java/org/astraea/app/common/DataRate.java index 3a010f47fe..85f0b61fc2 100644 --- a/app/src/main/java/org/astraea/app/common/DataRate.java +++ b/app/src/main/java/org/astraea/app/common/DataRate.java @@ -30,6 +30,42 @@ public class DataRate { private final DataSize totalBitTransmitted; private final BigDecimal durationInNanoSecond; + public static final DataRateSource Bit = (x) -> new DataRateBuilder(DataSize.Bit.of(x)); + public static final DataRateSource Kb = (x) -> new DataRateBuilder(DataSize.Kb.of(x)); + public static final DataRateSource Mb = (x) -> new DataRateBuilder(DataSize.Mb.of(x)); + public static final DataRateSource Gb = (x) -> new DataRateBuilder(DataSize.Gb.of(x)); + public static final DataRateSource Tb = (x) -> new DataRateBuilder(DataSize.Tb.of(x)); + public static final DataRateSource Pb = (x) -> new DataRateBuilder(DataSize.Pb.of(x)); + public static final DataRateSource Eb = (x) -> new DataRateBuilder(DataSize.Eb.of(x)); + public static final DataRateSource Yb = (x) -> new DataRateBuilder(DataSize.Yb.of(x)); + public static final DataRateSource Zb = (x) -> new DataRateBuilder(DataSize.Zb.of(x)); + public static final DataRateSource Kib = (x) -> new DataRateBuilder(DataSize.Kib.of(x)); + public static final DataRateSource Mib = (x) -> new DataRateBuilder(DataSize.Mib.of(x)); + public static final DataRateSource Gib = (x) -> new DataRateBuilder(DataSize.Gib.of(x)); + public static final DataRateSource Tib = (x) -> new DataRateBuilder(DataSize.Tib.of(x)); + public static final DataRateSource Pib = (x) -> new DataRateBuilder(DataSize.Pib.of(x)); + public static final DataRateSource Eib = (x) -> new DataRateBuilder(DataSize.Eib.of(x)); + public static final DataRateSource Yib = (x) -> new DataRateBuilder(DataSize.Yib.of(x)); + public static final DataRateSource Zib = (x) -> new DataRateBuilder(DataSize.Zib.of(x)); + + public static final DataRateSource Byte = (x) -> new DataRateBuilder(DataSize.Byte.of(x)); + public static final DataRateSource KB = (x) -> new DataRateBuilder(DataSize.KB.of(x)); + public static final DataRateSource MB = (x) -> new DataRateBuilder(DataSize.MB.of(x)); + public static final DataRateSource GB = (x) -> new DataRateBuilder(DataSize.GB.of(x)); + public static final DataRateSource TB = (x) -> new DataRateBuilder(DataSize.TB.of(x)); + public static final DataRateSource PB = (x) -> new DataRateBuilder(DataSize.PB.of(x)); + public static final DataRateSource EB = (x) -> new DataRateBuilder(DataSize.EB.of(x)); + public static final DataRateSource YB = (x) -> new DataRateBuilder(DataSize.YB.of(x)); + public static final DataRateSource ZB = (x) -> new DataRateBuilder(DataSize.ZB.of(x)); + public static final DataRateSource KiB = (x) -> new DataRateBuilder(DataSize.KiB.of(x)); + public static final DataRateSource MiB = (x) -> new DataRateBuilder(DataSize.MiB.of(x)); + public static final DataRateSource GiB = (x) -> new DataRateBuilder(DataSize.GiB.of(x)); + public static final DataRateSource TiB = (x) -> new DataRateBuilder(DataSize.TiB.of(x)); + public static final DataRateSource PiB = (x) -> new DataRateBuilder(DataSize.PiB.of(x)); + public static final DataRateSource EiB = (x) -> new DataRateBuilder(DataSize.EiB.of(x)); + public static final DataRateSource YiB = (x) -> new DataRateBuilder(DataSize.YiB.of(x)); + public static final DataRateSource ZiB = (x) -> new DataRateBuilder(DataSize.ZiB.of(x)); + static BigDecimal fromDurationToBigDecimalSafely(Duration duration) { // It the given duration is extremely long(like 1000 years), it might overflow the long // maximum value in nano second unit. @@ -237,4 +273,42 @@ public String toString(ChronoUnit chronoUnit) { public String toString() { return toString(ChronoUnit.SECONDS); } + + @FunctionalInterface + public interface DataRateSource { + DataRateBuilder of(long measurement); + } + + public static final class DataRateBuilder { + + private final DataSize dataSize; + + DataRateBuilder(DataSize dataSize) { + this.dataSize = dataSize; + } + + public DataRate perSecond() { + return dataSize.dataRate(ChronoUnit.SECONDS); + } + + public DataRate perMinute() { + return dataSize.dataRate(ChronoUnit.MINUTES); + } + + public DataRate perHour() { + return dataSize.dataRate(ChronoUnit.HOURS); + } + + public DataRate perDay() { + return dataSize.dataRate(ChronoUnit.DAYS); + } + + public DataRate over(Duration duration) { + return dataSize.dataRate(duration); + } + + public DataRate over(ChronoUnit chronoUnit) { + return dataSize.dataRate(chronoUnit); + } + } } diff --git a/app/src/test/java/org/astraea/app/common/DataRateTest.java b/app/src/test/java/org/astraea/app/common/DataRateTest.java index 643d4f33fa..5ea77c95cf 100644 --- a/app/src/test/java/org/astraea/app/common/DataRateTest.java +++ b/app/src/test/java/org/astraea/app/common/DataRateTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.math.BigInteger; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.function.BiConsumer; @@ -142,4 +143,68 @@ void testFromDouble() { DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).byteRate(), DataRate.of(1024.0).byteRate()); } + + @Test + void testDataRateOf() { + BiConsumer test = + (bits, rateBuilder) -> { + var perSecond = rateBuilder.perSecond(); + var perMinute = rateBuilder.perMinute(); + var perHour = rateBuilder.perHour(); + var perDay = rateBuilder.perDay(); + var per2Sec = rateBuilder.over(Duration.ofSeconds(2)); + var perSecond2 = rateBuilder.over(ChronoUnit.SECONDS); + Assertions.assertEquals( + bits, perSecond.toBigDecimal(DataUnit.Bit, ChronoUnit.SECONDS).toBigInteger()); + Assertions.assertEquals( + bits.divide(BigInteger.valueOf(60)), + perMinute.toBigDecimal(DataUnit.Bit, ChronoUnit.SECONDS).toBigInteger()); + Assertions.assertEquals( + bits.divide(BigInteger.valueOf(60 * 60)), + perHour.toBigDecimal(DataUnit.Bit, ChronoUnit.SECONDS).toBigInteger()); + Assertions.assertEquals( + bits.divide(BigInteger.valueOf(60 * 60 * 24)), + perDay.toBigDecimal(DataUnit.Bit, ChronoUnit.SECONDS).toBigInteger()); + Assertions.assertEquals( + bits.divide(BigInteger.valueOf(2)), + per2Sec.toBigDecimal(DataUnit.Bit, ChronoUnit.SECONDS).toBigInteger()); + Assertions.assertEquals( + bits, perSecond2.toBigDecimal(DataUnit.Bit, ChronoUnit.SECONDS).toBigInteger()); + }; + + test.accept(new BigInteger("1"), DataRate.Bit.of(1)); + test.accept(new BigInteger("1000"), DataRate.Kb.of(1)); + test.accept(new BigInteger("1000000"), DataRate.Mb.of(1)); + test.accept(new BigInteger("1000000000"), DataRate.Gb.of(1)); + test.accept(new BigInteger("1000000000000"), DataRate.Tb.of(1)); + test.accept(new BigInteger("1000000000000000"), DataRate.Pb.of(1)); + test.accept(new BigInteger("1000000000000000000"), DataRate.Eb.of(1)); + test.accept(new BigInteger("1000000000000000000000"), DataRate.Zb.of(1)); + test.accept(new BigInteger("1000000000000000000000000"), DataRate.Yb.of(1)); + test.accept(new BigInteger("1024"), DataRate.Kib.of(1)); + test.accept(new BigInteger("1048576"), DataRate.Mib.of(1)); + test.accept(new BigInteger("1073741824"), DataRate.Gib.of(1)); + test.accept(new BigInteger("1099511627776"), DataRate.Tib.of(1)); + test.accept(new BigInteger("1125899906842624"), DataRate.Pib.of(1)); + test.accept(new BigInteger("1152921504606846976"), DataRate.Eib.of(1)); + test.accept(new BigInteger("1180591620717411303424"), DataRate.Zib.of(1)); + test.accept(new BigInteger("1208925819614629174706176"), DataRate.Yib.of(1)); + test.accept(new BigInteger("8"), DataRate.Byte.of(1)); + test.accept(new BigInteger("8000"), DataRate.KB.of(1)); + test.accept(new BigInteger("8000000"), DataRate.MB.of(1)); + test.accept(new BigInteger("8000000000"), DataRate.GB.of(1)); + test.accept(new BigInteger("8000000000000"), DataRate.TB.of(1)); + test.accept(new BigInteger("8000000000000000"), DataRate.PB.of(1)); + test.accept(new BigInteger("8000000000000000000"), DataRate.EB.of(1)); + test.accept(new BigInteger("8000000000000000000000"), DataRate.ZB.of(1)); + test.accept(new BigInteger("8000000000000000000000000"), DataRate.YB.of(1)); + test.accept(new BigInteger("8192"), DataRate.KiB.of(1)); + test.accept(new BigInteger("8388608"), DataRate.MiB.of(1)); + test.accept(new BigInteger("8589934592"), DataRate.GiB.of(1)); + test.accept(new BigInteger("8796093022208"), DataRate.TiB.of(1)); + test.accept(new BigInteger("9007199254740992"), DataRate.PiB.of(1)); + test.accept(new BigInteger("9223372036854775808"), DataRate.EiB.of(1)); + test.accept(new BigInteger("9444732965739290427392"), DataRate.ZiB.of(1)); + test.accept(new BigInteger("9671406556917033397649408"), DataRate.YiB.of(1)); + } } From 9ba53f639a6c278ac174ddf8b74b42a5010dc549 Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Mon, 18 Jul 2022 01:27:54 +0800 Subject: [PATCH 12/18] nobody use these method --- .../java/org/astraea/app/common/DataRate.java | 32 ------------------- .../org/astraea/app/common/DataSizeTest.java | 11 ------- 2 files changed, 43 deletions(-) diff --git a/app/src/main/java/org/astraea/app/common/DataRate.java b/app/src/main/java/org/astraea/app/common/DataRate.java index 85f0b61fc2..a21a372cbc 100644 --- a/app/src/main/java/org/astraea/app/common/DataRate.java +++ b/app/src/main/java/org/astraea/app/common/DataRate.java @@ -198,38 +198,6 @@ public static DataRate of(DataSize dataSize, Duration duration) { return new DataRate(dataSize, duration); } - public static BigDecimal ofBigDecimal(long measurement, DataUnit unit, ChronoUnit time) { - return ofBigDecimal(unit.of(measurement), unit, time.getDuration()); - } - - public static BigDecimal ofBigDecimal(long measurement, DataUnit unit, Duration time) { - return ofBigDecimal(unit.of(measurement), unit, time); - } - - public static BigDecimal ofBigDecimal(DataSize size, DataUnit unit, ChronoUnit time) { - return ofBigDecimal(size, unit, time.getDuration()); - } - - public static BigDecimal ofBigDecimal(DataSize size, DataUnit unit, Duration time) { - return of(size, time).toBigDecimal(unit, time); - } - - public static double ofDouble(long measurement, DataUnit unit, ChronoUnit time) { - return ofBigDecimal(measurement, unit, time).doubleValue(); - } - - public static double ofDouble(long measurement, DataUnit unit, Duration time) { - return ofBigDecimal(measurement, unit, time).doubleValue(); - } - - public static double ofDouble(DataSize size, DataUnit unit, ChronoUnit time) { - return ofBigDecimal(size, unit, time).doubleValue(); - } - - public static double ofDouble(DataSize size, DataUnit unit, Duration time) { - return ofBigDecimal(size, unit, time).doubleValue(); - } - /** * @param bytesPerSecond the double value that represent a data rate in bytes/second unit * @return a {@link DataRate} converted from the given parameter. diff --git a/app/src/test/java/org/astraea/app/common/DataSizeTest.java b/app/src/test/java/org/astraea/app/common/DataSizeTest.java index 38c608c48d..31b71d775a 100644 --- a/app/src/test/java/org/astraea/app/common/DataSizeTest.java +++ b/app/src/test/java/org/astraea/app/common/DataSizeTest.java @@ -77,17 +77,6 @@ void typicalUsageOfDataUnit() { // faster convert between DataRate and others. var randomSize = DataSize.Byte.of(ThreadLocalRandom.current().nextLong()); - BigDecimal bigDecimal0 = DataRate.ofBigDecimal(1000, DataUnit.Byte, ChronoUnit.SECONDS); - BigDecimal bigDecimal1 = DataRate.ofBigDecimal(1000, DataUnit.Byte, Duration.ofSeconds(1)); - BigDecimal bigDecimal2 = DataRate.ofBigDecimal(randomSize, DataUnit.Byte, ChronoUnit.SECONDS); - BigDecimal bigDecimal3 = - DataRate.ofBigDecimal(randomSize, DataUnit.Byte, Duration.ofSeconds(1)); - - double double0 = DataRate.ofDouble(1000, DataUnit.Byte, ChronoUnit.SECONDS); - double double1 = DataRate.ofDouble(1000, DataUnit.Byte, Duration.ofSeconds(1)); - double double2 = DataRate.ofDouble(randomSize, DataUnit.Byte, ChronoUnit.SECONDS); - double double3 = DataRate.ofDouble(randomSize, DataUnit.Byte, Duration.ofSeconds(1)); - // sum all data size var sumAll = IntStream.range(0, 100).mapToObj(DataSize.Byte::of).reduce(DataSize.ZERO, DataSize::add); From 217870d89a7fd89d2a07c0983938b6603fe0c8f1 Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Mon, 18 Jul 2022 01:43:45 +0800 Subject: [PATCH 13/18] revise 1. add `DataRate.ZERO` 2. add `DataRate.Size.of()` 3. remove `DataRate.of(...)` --- .../app/admin/ReplicaSyncingMonitor.java | 2 +- .../java/org/astraea/app/common/DataRate.java | 136 ++++++------------ .../java/org/astraea/app/common/DataSize.java | 4 +- .../org/astraea/app/common/DataRateTest.java | 18 +-- .../org/astraea/app/common/DataSizeTest.java | 6 +- 5 files changed, 56 insertions(+), 110 deletions(-) diff --git a/app/src/main/java/org/astraea/app/admin/ReplicaSyncingMonitor.java b/app/src/main/java/org/astraea/app/admin/ReplicaSyncingMonitor.java index 20685c8f99..c9c3c5c391 100644 --- a/app/src/main/java/org/astraea/app/admin/ReplicaSyncingMonitor.java +++ b/app/src/main/java/org/astraea/app/admin/ReplicaSyncingMonitor.java @@ -252,7 +252,7 @@ public String progressBar() { public DataRate dataRate() { if (isProgressFallback()) { // log retention/compaction occurred, we don't know the actual data rate at this moment. - return DataRate.of(0, DataUnit.Byte, interval); + return DataRate.ZERO; } else { return currentSize.subtract(previousSize).dataRate(interval); } diff --git a/app/src/main/java/org/astraea/app/common/DataRate.java b/app/src/main/java/org/astraea/app/common/DataRate.java index a21a372cbc..56e7c20bc6 100644 --- a/app/src/main/java/org/astraea/app/common/DataRate.java +++ b/app/src/main/java/org/astraea/app/common/DataRate.java @@ -30,41 +30,44 @@ public class DataRate { private final DataSize totalBitTransmitted; private final BigDecimal durationInNanoSecond; - public static final DataRateSource Bit = (x) -> new DataRateBuilder(DataSize.Bit.of(x)); - public static final DataRateSource Kb = (x) -> new DataRateBuilder(DataSize.Kb.of(x)); - public static final DataRateSource Mb = (x) -> new DataRateBuilder(DataSize.Mb.of(x)); - public static final DataRateSource Gb = (x) -> new DataRateBuilder(DataSize.Gb.of(x)); - public static final DataRateSource Tb = (x) -> new DataRateBuilder(DataSize.Tb.of(x)); - public static final DataRateSource Pb = (x) -> new DataRateBuilder(DataSize.Pb.of(x)); - public static final DataRateSource Eb = (x) -> new DataRateBuilder(DataSize.Eb.of(x)); - public static final DataRateSource Yb = (x) -> new DataRateBuilder(DataSize.Yb.of(x)); - public static final DataRateSource Zb = (x) -> new DataRateBuilder(DataSize.Zb.of(x)); - public static final DataRateSource Kib = (x) -> new DataRateBuilder(DataSize.Kib.of(x)); - public static final DataRateSource Mib = (x) -> new DataRateBuilder(DataSize.Mib.of(x)); - public static final DataRateSource Gib = (x) -> new DataRateBuilder(DataSize.Gib.of(x)); - public static final DataRateSource Tib = (x) -> new DataRateBuilder(DataSize.Tib.of(x)); - public static final DataRateSource Pib = (x) -> new DataRateBuilder(DataSize.Pib.of(x)); - public static final DataRateSource Eib = (x) -> new DataRateBuilder(DataSize.Eib.of(x)); - public static final DataRateSource Yib = (x) -> new DataRateBuilder(DataSize.Yib.of(x)); - public static final DataRateSource Zib = (x) -> new DataRateBuilder(DataSize.Zib.of(x)); - - public static final DataRateSource Byte = (x) -> new DataRateBuilder(DataSize.Byte.of(x)); - public static final DataRateSource KB = (x) -> new DataRateBuilder(DataSize.KB.of(x)); - public static final DataRateSource MB = (x) -> new DataRateBuilder(DataSize.MB.of(x)); - public static final DataRateSource GB = (x) -> new DataRateBuilder(DataSize.GB.of(x)); - public static final DataRateSource TB = (x) -> new DataRateBuilder(DataSize.TB.of(x)); - public static final DataRateSource PB = (x) -> new DataRateBuilder(DataSize.PB.of(x)); - public static final DataRateSource EB = (x) -> new DataRateBuilder(DataSize.EB.of(x)); - public static final DataRateSource YB = (x) -> new DataRateBuilder(DataSize.YB.of(x)); - public static final DataRateSource ZB = (x) -> new DataRateBuilder(DataSize.ZB.of(x)); - public static final DataRateSource KiB = (x) -> new DataRateBuilder(DataSize.KiB.of(x)); - public static final DataRateSource MiB = (x) -> new DataRateBuilder(DataSize.MiB.of(x)); - public static final DataRateSource GiB = (x) -> new DataRateBuilder(DataSize.GiB.of(x)); - public static final DataRateSource TiB = (x) -> new DataRateBuilder(DataSize.TiB.of(x)); - public static final DataRateSource PiB = (x) -> new DataRateBuilder(DataSize.PiB.of(x)); - public static final DataRateSource EiB = (x) -> new DataRateBuilder(DataSize.EiB.of(x)); - public static final DataRateSource YiB = (x) -> new DataRateBuilder(DataSize.YiB.of(x)); - public static final DataRateSource ZiB = (x) -> new DataRateBuilder(DataSize.ZiB.of(x)); + public static final DataRate ZERO = DataRate.of(0); + + public static final DataSizeSource Size = DataRateBuilder::new; + public static final LongSource Bit = (x) -> new DataRateBuilder(DataSize.Bit.of(x)); + public static final LongSource Kb = (x) -> new DataRateBuilder(DataSize.Kb.of(x)); + public static final LongSource Mb = (x) -> new DataRateBuilder(DataSize.Mb.of(x)); + public static final LongSource Gb = (x) -> new DataRateBuilder(DataSize.Gb.of(x)); + public static final LongSource Tb = (x) -> new DataRateBuilder(DataSize.Tb.of(x)); + public static final LongSource Pb = (x) -> new DataRateBuilder(DataSize.Pb.of(x)); + public static final LongSource Eb = (x) -> new DataRateBuilder(DataSize.Eb.of(x)); + public static final LongSource Yb = (x) -> new DataRateBuilder(DataSize.Yb.of(x)); + public static final LongSource Zb = (x) -> new DataRateBuilder(DataSize.Zb.of(x)); + public static final LongSource Kib = (x) -> new DataRateBuilder(DataSize.Kib.of(x)); + public static final LongSource Mib = (x) -> new DataRateBuilder(DataSize.Mib.of(x)); + public static final LongSource Gib = (x) -> new DataRateBuilder(DataSize.Gib.of(x)); + public static final LongSource Tib = (x) -> new DataRateBuilder(DataSize.Tib.of(x)); + public static final LongSource Pib = (x) -> new DataRateBuilder(DataSize.Pib.of(x)); + public static final LongSource Eib = (x) -> new DataRateBuilder(DataSize.Eib.of(x)); + public static final LongSource Yib = (x) -> new DataRateBuilder(DataSize.Yib.of(x)); + public static final LongSource Zib = (x) -> new DataRateBuilder(DataSize.Zib.of(x)); + + public static final LongSource Byte = (x) -> new DataRateBuilder(DataSize.Byte.of(x)); + public static final LongSource KB = (x) -> new DataRateBuilder(DataSize.KB.of(x)); + public static final LongSource MB = (x) -> new DataRateBuilder(DataSize.MB.of(x)); + public static final LongSource GB = (x) -> new DataRateBuilder(DataSize.GB.of(x)); + public static final LongSource TB = (x) -> new DataRateBuilder(DataSize.TB.of(x)); + public static final LongSource PB = (x) -> new DataRateBuilder(DataSize.PB.of(x)); + public static final LongSource EB = (x) -> new DataRateBuilder(DataSize.EB.of(x)); + public static final LongSource YB = (x) -> new DataRateBuilder(DataSize.YB.of(x)); + public static final LongSource ZB = (x) -> new DataRateBuilder(DataSize.ZB.of(x)); + public static final LongSource KiB = (x) -> new DataRateBuilder(DataSize.KiB.of(x)); + public static final LongSource MiB = (x) -> new DataRateBuilder(DataSize.MiB.of(x)); + public static final LongSource GiB = (x) -> new DataRateBuilder(DataSize.GiB.of(x)); + public static final LongSource TiB = (x) -> new DataRateBuilder(DataSize.TiB.of(x)); + public static final LongSource PiB = (x) -> new DataRateBuilder(DataSize.PiB.of(x)); + public static final LongSource EiB = (x) -> new DataRateBuilder(DataSize.EiB.of(x)); + public static final LongSource YiB = (x) -> new DataRateBuilder(DataSize.YiB.of(x)); + public static final LongSource ZiB = (x) -> new DataRateBuilder(DataSize.ZiB.of(x)); static BigDecimal fromDurationToBigDecimalSafely(Duration duration) { // It the given duration is extremely long(like 1000 years), it might overflow the long @@ -142,62 +145,6 @@ public DataSize dataSize() { return new DataSize(bitsPerSecond); } - /** - * Create a {@link DataRate} object from data volume and time unit - * - * 
{@code
-   * DataRate.of(100, DataUnit.KB, ChronoUnit.SECONDS);    // 100 KB/s
-   * DataRate.of(100, DataUnit.MB, ChronoUnit.MONTHS);     // 50 MB/month
-   * }
- * - * @return an object {@link DataRate} correspond to given arguments - */ - public static DataRate of(long measurement, DataUnit dataUnit, ChronoUnit chronoUnit) { - return of(measurement, dataUnit, chronoUnit.getDuration()); - } - - /** - * Create a {@link DataRate} object from data volume and time duration - * - * 
{@code
-   * DataRate.of(100, DataUnit.KB, Duration.ofSeconds(1)); // 100 KB/s
-   * DataRate.of(100, DataUnit.MB, Duration.ofSeconds(2)); // 50 MB/s
-   * }
- * - * @return an object {@link DataRate} correspond to given arguments - */ - public static DataRate of(long measurement, DataUnit dataUnit, Duration duration) { - return new DataRate(dataUnit.of(measurement), duration); - } - - /** - * Create a {@link DataRate} object from data volume and time duration - * - * 
{@code
-   * DataRate.of(DataUnit.KB.of(100), ChronoUnit.SECONDS); // 100 KB/s
-   * DataRate.of(DataUnit.MB.of(100), ChronoUnit.SECONDS); // 50 MB/s
-   * }
- * - * @return an object {@link DataRate} correspond to given arguments - */ - public static DataRate of(DataSize dataSize, ChronoUnit chronoUnit) { - return new DataRate(dataSize, chronoUnit.getDuration()); - } - - /** - * Create a {@link DataRate} object from data volume and time duration - * - * 
{@code
-   * DataRate.of(DataUnit.KB.of(100), Duration.ofSeconds(1)); // 100 KB/s
-   * DataRate.of(DataUnit.MB.of(100), Duration.ofSeconds(2)); // 50 MB/s
-   * }
- * - * @return an object {@link DataRate} correspond to given arguments - */ - public static DataRate of(DataSize dataSize, Duration duration) { - return new DataRate(dataSize, duration); - } - /** * @param bytesPerSecond the double value that represent a data rate in bytes/second unit * @return a {@link DataRate} converted from the given parameter. @@ -243,10 +190,15 @@ public String toString() { } @FunctionalInterface - public interface DataRateSource { + public interface LongSource { DataRateBuilder of(long measurement); } + @FunctionalInterface + public interface DataSizeSource { + DataRateBuilder of(DataSize size); + } + public static final class DataRateBuilder { private final DataSize dataSize; diff --git a/app/src/main/java/org/astraea/app/common/DataSize.java b/app/src/main/java/org/astraea/app/common/DataSize.java index a495f413da..c09d3fc85a 100644 --- a/app/src/main/java/org/astraea/app/common/DataSize.java +++ b/app/src/main/java/org/astraea/app/common/DataSize.java @@ -254,14 +254,14 @@ public DataUnit idealDataUnit() { /** Return a {@link DataRate} based on current data size over a specific time unit. */ public DataRate dataRate(ChronoUnit chronoUnit) { - return DataRate.of(this, chronoUnit.getDuration()); + return DataRate.Size.of(this).over(chronoUnit); } /** * Return a {@link DataRate} based on current data size over a specific {@link Duration} of time. */ public DataRate dataRate(Duration timePassed) { - return DataRate.of(this, timePassed); + return DataRate.Size.of(this).over(timePassed); } /** @return a {@link DataRate} based on current data size over one second. */ diff --git a/app/src/test/java/org/astraea/app/common/DataRateTest.java b/app/src/test/java/org/astraea/app/common/DataRateTest.java index 5ea77c95cf..95faa44d1c 100644 --- a/app/src/test/java/org/astraea/app/common/DataRateTest.java +++ b/app/src/test/java/org/astraea/app/common/DataRateTest.java @@ -55,7 +55,7 @@ void fromDurationSafely() { @Test void idealDataRateAndUnit() { - var sut = DataRate.of(1000, DataUnit.KB, Duration.ofSeconds(1)); + var sut = DataRate.KB.of(1000).perSecond(); assertFloatingValueEquals(1.0, sut.idealDataRate(ChronoUnit.SECONDS).doubleValue()); assertSame(DataUnit.MB, sut.idealDataUnit(ChronoUnit.SECONDS)); @@ -63,12 +63,8 @@ void idealDataRateAndUnit() { @Test void dataRate() { - var sut0 = - DataRate.of(500, DataUnit.KB, Duration.ofSeconds(1)) - .toBigDecimal(DataUnit.MB, Duration.ofSeconds(1)); - var sut1 = - DataRate.of(500, DataUnit.KB, ChronoUnit.SECONDS) - .toBigDecimal(DataUnit.MB, Duration.ofSeconds(1)); + var sut0 = DataRate.KB.of(500).perSecond().toBigDecimal(DataUnit.MB, Duration.ofSeconds(1)); + var sut1 = DataRate.KB.of(500).perSecond().toBigDecimal(DataUnit.MB, Duration.ofSeconds(1)); assertFloatingValueEquals(0.5, sut0.doubleValue()); assertFloatingValueEquals(0.5, sut1.doubleValue()); @@ -97,7 +93,7 @@ void testDataRate( long passedSecond, double expectedIdealDataRate, DataUnit expectedDataUnit) { - DataRate sut = DataRate.of(measurement, dataUnit, Duration.ofSeconds(passedSecond)); + DataRate sut = dataUnit.of(measurement).dataRate(Duration.ofSeconds(passedSecond)); assertFloatingValueEquals( expectedIdealDataRate, sut.idealDataRate(ChronoUnit.SECONDS).doubleValue()); @@ -133,15 +129,13 @@ void testLongByteRate() { @Test void testFromLong() { Assertions.assertEquals( - DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).byteRate(), - DataRate.of(1024).byteRate()); + DataRate.Byte.of(1024).perSecond().byteRate(), DataRate.of(1024).byteRate()); } @Test void testFromDouble() { Assertions.assertEquals( - DataRate.of(1024, DataUnit.Byte, ChronoUnit.SECONDS).byteRate(), - DataRate.of(1024.0).byteRate()); + DataRate.Byte.of(1024).perSecond().byteRate(), DataRate.of(1024.0).byteRate()); } @Test diff --git a/app/src/test/java/org/astraea/app/common/DataSizeTest.java b/app/src/test/java/org/astraea/app/common/DataSizeTest.java index 31b71d775a..30d222b35b 100644 --- a/app/src/test/java/org/astraea/app/common/DataSizeTest.java +++ b/app/src/test/java/org/astraea/app/common/DataSizeTest.java @@ -86,9 +86,9 @@ void typicalUsageOfDataUnit() { long bytesInLong = randomSize.bytes(); long bytes10Gib = DataSize.Gib.of(10).bytes(); - // fast way & normal way to get second rate - DataRate faster = DataSize.Byte.of(1000).perSecond(); - DataRate normal = DataRate.of(1000, DataUnit.Byte, ChronoUnit.SECONDS); + // two ways to get second rate + DataRate one = DataSize.Byte.of(1000).perSecond(); + DataRate two = DataRate.Byte.of(1000).perSecond(); // data rate to other types double dataRateDouble = DataSize.Byte.of(1000).perSecond().byteRate(); From 4a7073c66d7775e34e33f7ac2a4b2c717a4f146e Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Mon, 18 Jul 2022 01:49:42 +0800 Subject: [PATCH 14/18] fix stack overflow --- app/src/main/java/org/astraea/app/common/DataSize.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/org/astraea/app/common/DataSize.java b/app/src/main/java/org/astraea/app/common/DataSize.java index c09d3fc85a..d3ea026cae 100644 --- a/app/src/main/java/org/astraea/app/common/DataSize.java +++ b/app/src/main/java/org/astraea/app/common/DataSize.java @@ -254,14 +254,14 @@ public DataUnit idealDataUnit() { /** Return a {@link DataRate} based on current data size over a specific time unit. */ public DataRate dataRate(ChronoUnit chronoUnit) { - return DataRate.Size.of(this).over(chronoUnit); + return dataRate(chronoUnit.getDuration()); } /** * Return a {@link DataRate} based on current data size over a specific {@link Duration} of time. */ public DataRate dataRate(Duration timePassed) { - return DataRate.Size.of(this).over(timePassed); + return new DataRate(this, timePassed); } /** @return a {@link DataRate} based on current data size over one second. */ From a31aff1176c2bf2f7269309c1cea80a9f7a7ce88 Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Mon, 18 Jul 2022 01:52:00 +0800 Subject: [PATCH 15/18] test --- .../test/java/org/astraea/app/common/DataRateTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/src/test/java/org/astraea/app/common/DataRateTest.java b/app/src/test/java/org/astraea/app/common/DataRateTest.java index 95faa44d1c..777f367ab5 100644 --- a/app/src/test/java/org/astraea/app/common/DataRateTest.java +++ b/app/src/test/java/org/astraea/app/common/DataRateTest.java @@ -200,5 +200,13 @@ void testDataRateOf() { test.accept(new BigInteger("9223372036854775808"), DataRate.EiB.of(1)); test.accept(new BigInteger("9444732965739290427392"), DataRate.ZiB.of(1)); test.accept(new BigInteger("9671406556917033397649408"), DataRate.YiB.of(1)); + + DataSize size = DataSize.GB.of(1024); + test.accept(size.bits(), DataRate.Size.of(size)); + } + + @Test + void testZero() { + Assertions.assertEquals(0, DataRate.ZERO.byteRate()); } } From 39054c541c2bff5aa493ceec004c0b616dcb7d79 Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Mon, 18 Jul 2022 01:55:47 +0800 Subject: [PATCH 16/18] fix merge --- .../astraea/app/performance/Performance.java | 6 +-- .../app/performance/DataSupplierTest.java | 39 ++++++++++--------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/app/src/main/java/org/astraea/app/performance/Performance.java b/app/src/main/java/org/astraea/app/performance/Performance.java index 6f23073ec4..ff53243945 100644 --- a/app/src/main/java/org/astraea/app/performance/Performance.java +++ b/app/src/main/java/org/astraea/app/performance/Performance.java @@ -338,13 +338,13 @@ Isolation isolation() { names = {"--key.size"}, description = "DataSize of the key. Default: 4Byte", converter = DataSize.Field.class) - DataSize keySize = DataUnit.Byte.of(4); + DataSize keySize = DataSize.Byte.of(4); @Parameter( names = {"--value.size"}, description = "DataSize of the value. Default: 1KiB", converter = DataSize.Field.class) - DataSize valueSize = DataUnit.KiB.of(1); + DataSize valueSize = DataSize.KiB.of(1); @Parameter( names = {"--key.distribution"}, @@ -371,7 +371,7 @@ Isolation isolation() { names = {"--throughput"}, description = "dataSize: size output per second. e.g. \"500KiB\"", converter = DataSize.Field.class) - DataSize throughput = DataUnit.GiB.of(500); + DataSize throughput = DataSize.GiB.of(500); @Parameter( names = {"--report.path"}, diff --git a/app/src/test/java/org/astraea/app/performance/DataSupplierTest.java b/app/src/test/java/org/astraea/app/performance/DataSupplierTest.java index c197dfa91d..7e85482e74 100644 --- a/app/src/test/java/org/astraea/app/performance/DataSupplierTest.java +++ b/app/src/test/java/org/astraea/app/performance/DataSupplierTest.java @@ -19,6 +19,7 @@ import java.time.Duration; import java.util.Arrays; import java.util.concurrent.atomic.AtomicLong; +import org.astraea.app.common.DataSize; import org.astraea.app.common.DataUnit; import org.astraea.app.common.Utils; import org.junit.jupiter.api.Assertions; @@ -30,7 +31,7 @@ public class DataSupplierTest { void testDuration() { var dataSupplier = DataSupplier.of( - ExeTime.of("2s"), () -> 1L, () -> 4L, () -> 1L, () -> 10L, DataUnit.KiB.of(100)); + ExeTime.of("2s"), () -> 1L, () -> 4L, () -> 1L, () -> 10L, DataSize.KiB.of(100)); Assertions.assertTrue(dataSupplier.get().hasData()); Utils.sleep(Duration.ofSeconds(3)); Assertions.assertFalse(dataSupplier.get().hasData()); @@ -41,11 +42,11 @@ void testRecordLimit() { var dataSupplier = DataSupplier.of( ExeTime.of("2records"), - () -> DataUnit.KiB.of(2).measurement(DataUnit.Byte).longValue(), + () -> DataSize.KiB.of(2).measurement(DataUnit.Byte).longValue(), () -> 10L, - () -> DataUnit.KiB.of(100).measurement(DataUnit.Byte).longValue(), + () -> DataSize.KiB.of(100).measurement(DataUnit.Byte).longValue(), () -> 10L, - DataUnit.KiB.of(102)); + DataSize.KiB.of(102)); Assertions.assertTrue(dataSupplier.get().hasData()); Assertions.assertTrue(dataSupplier.get().hasData()); Assertions.assertFalse(dataSupplier.get().hasData()); @@ -57,10 +58,10 @@ void testKeySize() { DataSupplier.of( ExeTime.of("10s"), () -> 1L, - () -> DataUnit.Byte.of(20).measurement(DataUnit.Byte).longValue(), + () -> DataSize.Byte.of(20).measurement(DataUnit.Byte).longValue(), () -> 2L, - () -> DataUnit.KiB.of(100).measurement(DataUnit.Byte).longValue(), - DataUnit.KiB.of(200)); + () -> DataSize.KiB.of(100).measurement(DataUnit.Byte).longValue(), + DataSize.KiB.of(200)); var data1 = dataSupplier.get(); Assertions.assertTrue(data1.hasData()); var data2 = dataSupplier.get(); @@ -75,10 +76,10 @@ void testFixedValueSize() { DataSupplier.of( ExeTime.of("10s"), () -> 1L, - () -> DataUnit.Byte.of(20).measurement(DataUnit.Byte).longValue(), + () -> DataSize.Byte.of(20).measurement(DataUnit.Byte).longValue(), () -> 2L, - () -> DataUnit.KiB.of(100).measurement(DataUnit.Byte).longValue(), - DataUnit.KiB.of(100)); + () -> DataSize.KiB.of(100).measurement(DataUnit.Byte).longValue(), + DataSize.KiB.of(100)); var data = dataSupplier.get(); Assertions.assertTrue(data.hasData()); // initial value size is 100KB and the distributed is fixed to zero, so the final size is 102400 @@ -98,7 +99,7 @@ void testKeyDistribution() { () -> 100L + counter2.getAndIncrement(), () -> 10L, () -> 10L, - DataUnit.KiB.of(200)); + DataSize.KiB.of(200)); var data1 = dataSupplier.get(); Assertions.assertTrue(data1.hasData()); var data2 = dataSupplier.get(); @@ -120,7 +121,7 @@ void testKeyDistribution() { () -> 100L, () -> 10L, () -> 10L, - DataUnit.KiB.of(200)); + DataSize.KiB.of(200)); data1 = dataSupplier.get(); Assertions.assertTrue(data1.hasData()); data2 = dataSupplier.get(); @@ -144,7 +145,7 @@ void testDistributedValueSize() { () -> 10L, () -> counter.getAndIncrement() % 2, () -> 100L + counter2.getAndIncrement(), - DataUnit.KiB.of(100)); + DataSize.KiB.of(100)); var data1 = dataSupplier.get(); Assertions.assertTrue(data1.hasData()); var data2 = dataSupplier.get(); @@ -166,7 +167,7 @@ void testDistributedValueSize() { () -> 10L, () -> counter.getAndIncrement() % 2, () -> 100L, - DataUnit.KiB.of(100)); + DataSize.KiB.of(100)); data1 = dataSupplier.get(); Assertions.assertTrue(data1.hasData()); data2 = dataSupplier.get(); @@ -183,10 +184,10 @@ void testThrottle() { DataSupplier.of( ExeTime.of("10s"), () -> 10L, - () -> DataUnit.KiB.of(50).measurement(DataUnit.Byte).longValue(), + () -> DataSize.KiB.of(50).measurement(DataUnit.Byte).longValue(), () -> 10L, - () -> DataUnit.KiB.of(50).measurement(DataUnit.Byte).longValue(), - DataUnit.KiB.of(150)); + () -> DataSize.KiB.of(50).measurement(DataUnit.Byte).longValue(), + DataSize.KiB.of(150)); // total: 100KB, limit: 150KB -> no throttle Assertions.assertTrue(dataSupplier.get().hasData()); // total: 200KB, limit: 150KB -> will throttle next data @@ -199,7 +200,7 @@ void testThrottle() { void testNoKey() { var dataSupplier = DataSupplier.of( - ExeTime.of("10s"), () -> 10L, () -> 0L, () -> 10L, () -> 10L, DataUnit.KiB.of(200)); + ExeTime.of("10s"), () -> 10L, () -> 0L, () -> 10L, () -> 10L, DataSize.KiB.of(200)); var data = dataSupplier.get(); Assertions.assertTrue(data.hasData()); @@ -210,7 +211,7 @@ void testNoKey() { void testNoValue() { var dataSupplier = DataSupplier.of( - ExeTime.of("10s"), () -> 10L, () -> 10L, () -> 10L, () -> 0L, DataUnit.KiB.of(200)); + ExeTime.of("10s"), () -> 10L, () -> 10L, () -> 10L, () -> 0L, DataSize.KiB.of(200)); var data = dataSupplier.get(); Assertions.assertTrue(data.hasData()); Assertions.assertNull(data.value()); From bd20bacb0a893aa4e50aace4b0258ae729bcc6a7 Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Mon, 18 Jul 2022 12:24:33 +0800 Subject: [PATCH 17/18] remove --- .../java/org/astraea/app/common/DataRate.java | 28 +------------------ .../java/org/astraea/app/common/DataSize.java | 5 ---- .../org/astraea/app/common/DataRateTest.java | 26 +++++------------ .../org/astraea/app/common/DataSizeTest.java | 21 ++------------ 4 files changed, 10 insertions(+), 70 deletions(-) diff --git a/app/src/main/java/org/astraea/app/common/DataRate.java b/app/src/main/java/org/astraea/app/common/DataRate.java index 56e7c20bc6..af2b5b3303 100644 --- a/app/src/main/java/org/astraea/app/common/DataRate.java +++ b/app/src/main/java/org/astraea/app/common/DataRate.java @@ -30,7 +30,7 @@ public class DataRate { private final DataSize totalBitTransmitted; private final BigDecimal durationInNanoSecond; - public static final DataRate ZERO = DataRate.of(0); + public static final DataRate ZERO = DataRate.Byte.of(0).perSecond(); public static final DataSizeSource Size = DataRateBuilder::new; public static final LongSource Bit = (x) -> new DataRateBuilder(DataSize.Bit.of(x)); @@ -145,32 +145,6 @@ public DataSize dataSize() { return new DataSize(bitsPerSecond); } - /** - * @param bytesPerSecond the double value that represent a data rate in bytes/second unit - * @return a {@link DataRate} converted from the given parameter. - */ - public static DataRate of(double bytesPerSecond) { - var bits = - BigDecimal.valueOf(bytesPerSecond) - .multiply(new BigDecimal(DataUnit.Byte.bits)) - .toBigInteger(); - var size = new DataSize(bits); - return new DataRate(size, Duration.ofSeconds(1)); - } - - /** - * @param bytesPerSecond the double value that represent a data rate in bytes/second unit - * @return a {@link DataRate} converted from the given parameter. - */ - public static DataRate of(long bytesPerSecond) { - var bits = - BigDecimal.valueOf(bytesPerSecond) - .multiply(new BigDecimal(DataUnit.Byte.bits)) - .toBigInteger(); - var size = new DataSize(bits); - return new DataRate(size, Duration.ofSeconds(1)); - } - private static String chronoName(ChronoUnit chronoUnit) { return chronoUnit.name().toLowerCase(Locale.ROOT); } diff --git a/app/src/main/java/org/astraea/app/common/DataSize.java b/app/src/main/java/org/astraea/app/common/DataSize.java index d3ea026cae..fff5274ddd 100644 --- a/app/src/main/java/org/astraea/app/common/DataSize.java +++ b/app/src/main/java/org/astraea/app/common/DataSize.java @@ -264,11 +264,6 @@ public DataRate dataRate(Duration timePassed) { return new DataRate(this, timePassed); } - /** @return a {@link DataRate} based on current data size over one second. */ - public DataRate perSecond() { - return dataRate(ChronoUnit.SECONDS); - } - /** Return a string represent current size in given data unit. */ public String toString(DataUnit unit) { var divide = diff --git a/app/src/test/java/org/astraea/app/common/DataRateTest.java b/app/src/test/java/org/astraea/app/common/DataRateTest.java index 777f367ab5..e9086bc985 100644 --- a/app/src/test/java/org/astraea/app/common/DataRateTest.java +++ b/app/src/test/java/org/astraea/app/common/DataRateTest.java @@ -102,9 +102,9 @@ void testDataRate( @Test void testToDataSize() { - Assertions.assertEquals(DataUnit.Byte.of(1024), DataUnit.Byte.of(1024).perSecond().dataSize()); - Assertions.assertEquals(DataUnit.KiB.of(1024), DataUnit.KiB.of(1024).perSecond().dataSize()); - Assertions.assertEquals(DataUnit.EiB.of(24), DataUnit.EiB.of(24).perSecond().dataSize()); + Assertions.assertEquals(DataUnit.Byte.of(1024), DataRate.Byte.of(1024).perSecond().dataSize()); + Assertions.assertEquals(DataUnit.KiB.of(1024), DataRate.KiB.of(1024).perSecond().dataSize()); + Assertions.assertEquals(DataUnit.EiB.of(24), DataRate.EiB.of(24).perSecond().dataSize()); } @Test @@ -116,26 +116,14 @@ void testDoubleByteRate() { "The value " + a + " and " + b + " should have no difference above 1e-8"); }; - assertDoubleEqual.accept(1024.0, DataUnit.Byte.of(1024).perSecond().byteRate()); - assertDoubleEqual.accept(1024.0 * 1024, DataUnit.KiB.of(1024).perSecond().byteRate()); + assertDoubleEqual.accept(1024.0, DataRate.Byte.of(1024).perSecond().byteRate()); + assertDoubleEqual.accept(1024.0 * 1024, DataRate.KiB.of(1024).perSecond().byteRate()); } @Test void testLongByteRate() { - Assertions.assertEquals(1024L, DataUnit.Byte.of(1024).perSecond().byteRate()); - Assertions.assertEquals(1024L * 1024, DataUnit.KiB.of(1024).perSecond().byteRate()); - } - - @Test - void testFromLong() { - Assertions.assertEquals( - DataRate.Byte.of(1024).perSecond().byteRate(), DataRate.of(1024).byteRate()); - } - - @Test - void testFromDouble() { - Assertions.assertEquals( - DataRate.Byte.of(1024).perSecond().byteRate(), DataRate.of(1024.0).byteRate()); + Assertions.assertEquals(1024L, DataRate.Byte.of(1024).perSecond().byteRate()); + Assertions.assertEquals(1024L * 1024, DataRate.KiB.of(1024).perSecond().byteRate()); } @Test diff --git a/app/src/test/java/org/astraea/app/common/DataSizeTest.java b/app/src/test/java/org/astraea/app/common/DataSizeTest.java index 30d222b35b..50129ba7df 100644 --- a/app/src/test/java/org/astraea/app/common/DataSizeTest.java +++ b/app/src/test/java/org/astraea/app/common/DataSizeTest.java @@ -87,18 +87,15 @@ void typicalUsageOfDataUnit() { long bytes10Gib = DataSize.Gib.of(10).bytes(); // two ways to get second rate - DataRate one = DataSize.Byte.of(1000).perSecond(); DataRate two = DataRate.Byte.of(1000).perSecond(); // data rate to other types - double dataRateDouble = DataSize.Byte.of(1000).perSecond().byteRate(); - DataSize dataRateSize = DataSize.Byte.of(1000).perSecond().dataSize(); + double dataRateDouble = DataRate.Byte.of(1000).perSecond().byteRate(); + DataSize dataRateSize = DataRate.Byte.of(1000).perSecond().dataSize(); // fast way to get DataSize & DataRate from primitive type DataSize primitive0 = DataSize.Byte.of(1000); DataSize primitive1 = DataSize.Byte.of((long) 1000.0); - DataRate primitive2 = DataRate.of(123.0); - DataRate primitive3 = DataRate.of(1024); // fast way to add/subtract data from primitive type DataSize.Byte.of(1000).subtract(500); @@ -375,20 +372,6 @@ void compare() { assertFalse(DataUnit.MB.of(1).greaterThan(DataUnit.KB.of(1001))); } - @ParameterizedTest - @CsvSource( - delimiterString = " ", - value = {"1000 KiB", "1234 Kib", "5566 GB", "42 Byte"}) - void perSecond(long measurement, DataUnit unit) { - var dataSize = unit.of(measurement); - var dataRate = dataSize.perSecond(); - - Assertions.assertEquals( - measurement, dataRate.toBigDecimal(unit, ChronoUnit.SECONDS).longValueExact()); - Assertions.assertEquals( - measurement * 60, dataRate.toBigDecimal(unit, ChronoUnit.MINUTES).longValueExact()); - } - @ParameterizedTest @CsvSource( delimiterString = ",", From e5530cac6737039dee7111965c8c1e3943b569ee Mon Sep 17 00:00:00 2001 From: Zheng-Xian Li Date: Mon, 18 Jul 2022 13:29:45 +0800 Subject: [PATCH 18/18] fix test --- app/src/main/java/org/astraea/app/common/DataRate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/org/astraea/app/common/DataRate.java b/app/src/main/java/org/astraea/app/common/DataRate.java index af2b5b3303..7f0b0d4b8c 100644 --- a/app/src/main/java/org/astraea/app/common/DataRate.java +++ b/app/src/main/java/org/astraea/app/common/DataRate.java @@ -30,8 +30,6 @@ public class DataRate { private final DataSize totalBitTransmitted; private final BigDecimal durationInNanoSecond; - public static final DataRate ZERO = DataRate.Byte.of(0).perSecond(); - public static final DataSizeSource Size = DataRateBuilder::new; public static final LongSource Bit = (x) -> new DataRateBuilder(DataSize.Bit.of(x)); public static final LongSource Kb = (x) -> new DataRateBuilder(DataSize.Kb.of(x)); @@ -69,6 +67,8 @@ public class DataRate { public static final LongSource YiB = (x) -> new DataRateBuilder(DataSize.YiB.of(x)); public static final LongSource ZiB = (x) -> new DataRateBuilder(DataSize.ZiB.of(x)); + public static final DataRate ZERO = DataRate.Byte.of(0).perSecond(); + static BigDecimal fromDurationToBigDecimalSafely(Duration duration) { // It the given duration is extremely long(like 1000 years), it might overflow the long // maximum value in nano second unit.