From 56891b7f1ee465a83cbb400b5fefc8341b6622e6 Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Tue, 28 Jan 2020 16:10:41 +0800 Subject: [PATCH 1/8] [ARROW-7610][Java] Finish support for 64 bit int allocations --- .../apache/arrow/memory/BaseAllocator.java | 2 +- .../DefaultAllocationManagerOption.java | 98 +++++++++ .../arrow/memory/NettyAllocationManager.java | 2 +- .../arrow/memory/UnsafeAllocationManager.java | 65 ++++++ .../arrow/memory/TestLargeArrowBuf.java | 68 +++++++ .../apache/arrow/vector/BaseValueVector.java | 20 +- .../org/apache/arrow/vector/BigIntVector.java | 10 +- .../apache/arrow/vector/DateDayVector.java | 10 +- .../apache/arrow/vector/DateMilliVector.java | 10 +- .../apache/arrow/vector/DecimalVector.java | 22 +- .../apache/arrow/vector/DurationVector.java | 8 +- .../arrow/vector/FixedSizeBinaryVector.java | 10 +- .../org/apache/arrow/vector/Float4Vector.java | 10 +- .../org/apache/arrow/vector/Float8Vector.java | 10 +- .../org/apache/arrow/vector/IntVector.java | 10 +- .../arrow/vector/IntervalDayVector.java | 16 +- .../arrow/vector/IntervalYearVector.java | 12 +- .../apache/arrow/vector/SmallIntVector.java | 12 +- .../apache/arrow/vector/TimeMicroVector.java | 10 +- .../apache/arrow/vector/TimeMilliVector.java | 10 +- .../apache/arrow/vector/TimeNanoVector.java | 10 +- .../apache/arrow/vector/TimeSecVector.java | 10 +- .../arrow/vector/TimeStampMicroTZVector.java | 4 +- .../arrow/vector/TimeStampMicroVector.java | 4 +- .../arrow/vector/TimeStampMilliTZVector.java | 4 +- .../arrow/vector/TimeStampMilliVector.java | 4 +- .../arrow/vector/TimeStampNanoTZVector.java | 4 +- .../arrow/vector/TimeStampNanoVector.java | 4 +- .../arrow/vector/TimeStampSecTZVector.java | 4 +- .../arrow/vector/TimeStampSecVector.java | 4 +- .../apache/arrow/vector/TimeStampVector.java | 6 +- .../org/apache/arrow/vector/UInt2Vector.java | 12 +- .../org/apache/arrow/vector/UInt4Vector.java | 10 +- .../org/apache/arrow/vector/UInt8Vector.java | 10 +- .../arrow/vector/util/DecimalUtility.java | 6 +- .../apache/arrow/vector/TestLargeVector.java | 188 ++++++++++++++++++ 36 files changed, 559 insertions(+), 140 deletions(-) create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java create mode 100644 java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java index 84a8f63e591..0346739fc21 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java @@ -765,7 +765,7 @@ abstract static class Config { */ @Value.Default AllocationManager.Factory getAllocationManagerFactory() { - return NettyAllocationManager.FACTORY; + return DefaultAllocationManagerOption.DEFAULT_ALLOCATION_MANAGER_FACTORY; } /** diff --git a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java new file mode 100644 index 00000000000..46a54734889 --- /dev/null +++ b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +/** + * A class for choosing the default allocation manager. + */ +public class DefaultAllocationManagerOption { + + /** + * The environmental variable to set the default allocation manager type. + */ + public static final String ALLOCATION_MANAGER_TYPE_ENV_NAME = "ARROW_ALLOCATION_MANAGER_TYPE"; + + /** + * The system property to set the default allocation manager type. + */ + public static final String ALLOCATION_MANAGER_TYPE_PROPERTY_NAME = "arrow.allocation.manager.type"; + + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DefaultAllocationManagerOption.class); + + /** + * The default allocation manager factory. + */ + public static final AllocationManager.Factory DEFAULT_ALLOCATION_MANAGER_FACTORY = + getDefaultAllocationManagerFactory(); + + /** + * The allocation manager type. + */ + public enum AllocationManagerType { + /** + * Netty based allocation manager. + */ + Netty, + + /** + * Unsafe based allocation manager. + */ + Unsafe, + + /** + * Unknown type. + */ + Unknown, + } + + private static AllocationManagerType getDefaultAllocationManagerType() { + AllocationManagerType ret = AllocationManagerType.Unknown; + + String envValue = System.getenv(ALLOCATION_MANAGER_TYPE_ENV_NAME); + if ("netty".equals(envValue)) { + ret = AllocationManagerType.Netty; + } else if ("unsafe".equals(envValue)) { + ret = AllocationManagerType.Unsafe; + } + + // system property takes precedence + String propValue = System.getProperty(ALLOCATION_MANAGER_TYPE_PROPERTY_NAME); + if ("netty".equals(propValue)) { + ret = AllocationManagerType.Netty; + } else if ("unsafe".equals(propValue)) { + ret = AllocationManagerType.Unsafe; + } + return ret; + } + + private static AllocationManager.Factory getDefaultAllocationManagerFactory() { + AllocationManagerType type = getDefaultAllocationManagerType(); + + switch (type) { + case Netty: + return NettyAllocationManager.FACTORY; + case Unsafe: + return UnsafeAllocationManager.FACTORY; + case Unknown: + logger.warn("allocation manager type not specified, using netty as the default type"); + return NettyAllocationManager.FACTORY; + default: + throw new IllegalStateException("Unknown allocation manager type: " + type); + } + } +} diff --git a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java index 35b02f54ecd..e6da8a2a8aa 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java @@ -23,7 +23,7 @@ import io.netty.buffer.UnsafeDirectLittleEndian; /** - * The default implementation of AllocationManagerBase. The implementation is responsible for managing when memory + * The default implementation of {@link AllocationManager}. The implementation is responsible for managing when memory * is allocated and returned to the Netty-based PooledByteBufAllocatorL. */ public class NettyAllocationManager extends AllocationManager { diff --git a/java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java new file mode 100644 index 00000000000..8d67919f7f1 --- /dev/null +++ b/java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import io.netty.util.internal.PlatformDependent; + +/** + * Allocation manager based on unsafe API. + */ +public class UnsafeAllocationManager extends AllocationManager { + + public static final Factory FACTORY = new Factory(); + + private final long allocatedSize; + + private final long allocatedAddress; + + protected UnsafeAllocationManager(BaseAllocator accountingAllocator, long requestedSize) { + super(accountingAllocator); + allocatedAddress = PlatformDependent.allocateMemory(requestedSize); + allocatedSize = requestedSize; + } + + @Override + public long getSize() { + return allocatedSize; + } + + @Override + protected long memoryAddress() { + return allocatedAddress; + } + + @Override + protected void release0() { + PlatformDependent.freeMemory(allocatedAddress); + } + + /** + * Factory for creating {@link UnsafeAllocationManager}. + */ + public static class Factory implements AllocationManager.Factory { + private Factory() {} + + @Override + public AllocationManager create(BaseAllocator accountingAllocator, long size) { + return new UnsafeAllocationManager(accountingAllocator, size); + } + } +} diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java b/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java new file mode 100644 index 00000000000..1448fcae122 --- /dev/null +++ b/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import static org.junit.Assert.assertEquals; + +import io.netty.buffer.ArrowBuf; + +/** + * Integration test for large (more than 2GB) {@link io.netty.buffer.ArrowBuf}. + * To run this test, please + *
  • Make sure there are 4GB memory available in the system.
    • + *
  • + * Make sure the default allocation manager type is unsafe. + * This can be achieved by the environmental variable or system property. + * The details can be found in {@link DefaultAllocationManagerOption}. + *
    • + */ +public class TestLargeArrowBuf { + + private static void testLargeArrowBuf() { + final long bufSize = 4 * 1024 * 1024 * 1024L; + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + ArrowBuf largeBuf = allocator.buffer(bufSize)) { + assertEquals(bufSize, largeBuf.capacity()); + System.out.println("Successfully allocated a buffer with capacity " + largeBuf.capacity()); + + for (long i = 0; i < bufSize / 8; i++) { + largeBuf.setLong(i * 8, i); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully written " + (i + 1) + " long words"); + } + } + System.out.println("Successfully written " + (bufSize / 8) + " long words"); + + for (long i = 0; i < bufSize / 8; i++) { + long val = largeBuf.getLong(i * 8); + assertEquals(i, val); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully read " + (i + 1) + " long words"); + } + } + System.out.println("Successfully read " + (bufSize / 8) + " long words"); + } + System.out.println("Successfully released the large buffer."); + } + + public static void main(String[] args) { + testLargeArrowBuf(); + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index ede382110de..d8684714e78 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -124,7 +124,7 @@ protected static int getValidityBufferSizeFromCount(final int valueCount) { } /* round up bytes for the validity buffer for the given valueCount */ - private static long roundUp8ForValidityBuffer(int valueCount) { + private static long roundUp8ForValidityBuffer(long valueCount) { return ((valueCount + 63) >> 6) << 3; } @@ -140,7 +140,7 @@ long computeCombinedBufferSize(int valueCount, int typeWidth) { // for boolean type, value-buffer and validity-buffer are of same size. bufferSize *= 2; } else { - bufferSize += DataSizeRoundingUtil.roundUpTo8Multiple(valueCount * typeWidth); + bufferSize += DataSizeRoundingUtil.roundUpTo8Multiple((long) valueCount * typeWidth); } return BaseAllocator.nextPowerOfTwo(bufferSize); } @@ -170,16 +170,16 @@ DataAndValidityBuffers allocFixedDataAndValidityBufs(int valueCount, int typeWid long bufferSize = computeCombinedBufferSize(valueCount, typeWidth); assert bufferSize <= MAX_ALLOCATION_SIZE; - int validityBufferSize; - int dataBufferSize; + long validityBufferSize; + long dataBufferSize; if (typeWidth == 0) { - validityBufferSize = dataBufferSize = (int) (bufferSize / 2); + validityBufferSize = dataBufferSize = bufferSize / 2; } else { // Due to roundup to power-of-2 allocation, the bufferSize could be greater than the // requested size. Utilize the allocated buffer fully.; - int actualCount = (int) ((bufferSize * 8.0) / (8 * typeWidth + 1)); + long actualCount = (long) ((bufferSize * 8.0) / (8 * typeWidth + 1)); do { - validityBufferSize = (int) roundUp8ForValidityBuffer(actualCount); + validityBufferSize = roundUp8ForValidityBuffer(actualCount); dataBufferSize = DataSizeRoundingUtil.roundUpTo8Multiple(actualCount * typeWidth); if (validityBufferSize + dataBufferSize <= bufferSize) { break; @@ -191,14 +191,14 @@ DataAndValidityBuffers allocFixedDataAndValidityBufs(int valueCount, int typeWid /* allocate combined buffer */ - ArrowBuf combinedBuffer = allocator.buffer((int) bufferSize); + ArrowBuf combinedBuffer = allocator.buffer(bufferSize); /* slice into requested lengths */ ArrowBuf dataBuf = null; ArrowBuf validityBuf = null; - int bufferOffset = 0; + long bufferOffset = 0; for (int numBuffers = 0; numBuffers < 2; ++numBuffers) { - int len = (numBuffers == 0 ? dataBufferSize : validityBufferSize); + long len = (numBuffers == 0 ? dataBufferSize : validityBufferSize); ArrowBuf buf = combinedBuffer.slice(bufferOffset, len); buf.getReferenceManager().retain(); buf.readerIndex(0); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java index ac2015ea4ef..b392eb4da88 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java @@ -112,7 +112,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -128,7 +128,7 @@ public void get(int index, NullableBigIntHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -141,7 +141,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } @@ -153,7 +153,7 @@ public Long getObject(int index) { private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** @@ -275,7 +275,7 @@ public void setSafe(int index, int isSet, long value) { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java index 9448a1a1453..dab08e03dfa 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java @@ -114,7 +114,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableDateDayHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } } @@ -155,7 +155,7 @@ public Integer getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** @@ -279,7 +279,7 @@ public void setSafe(int index, int isSet, int value) { * @return value stored at the index. */ public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java index d893ba4f331..c3037ef1a2f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java @@ -116,7 +116,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -132,7 +132,7 @@ public void get(int index, NullableDateMilliHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -145,7 +145,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long millis = valueBuffer.getLong(index * TYPE_WIDTH); + final long millis = valueBuffer.getLong((long) index * TYPE_WIDTH); return DateUtility.getLocalDateTimeFromEpochMilli(millis); } } @@ -158,7 +158,7 @@ public LocalDateTime getObject(int index) { private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** @@ -282,7 +282,7 @@ public void setSafe(int index, int isSet, long value) { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index 0dc1bdd3fb4..0cd9d139de5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -126,7 +126,7 @@ public ArrowBuf get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + return valueBuffer.slice((long) index * TYPE_WIDTH, TYPE_WIDTH); } /** @@ -192,7 +192,7 @@ public int getScale() { */ public void set(int index, ArrowBuf buffer) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); } /** @@ -215,9 +215,9 @@ public void setBigEndian(int index, byte[] value) { final int length = value.length; // do the bound check. - valueBuffer.checkBytes(index * TYPE_WIDTH, (index + 1) * TYPE_WIDTH); + valueBuffer.checkBytes((long) index * TYPE_WIDTH, (long) (index + 1) * TYPE_WIDTH); - long outAddress = valueBuffer.memoryAddress() + index * TYPE_WIDTH; + long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH; // swap bytes to convert BE to LE for (int byteIdx = 0; byteIdx < length; ++byteIdx) { PlatformDependent.putByte(outAddress + byteIdx, value[length - 1 - byteIdx]); @@ -248,7 +248,7 @@ public void setBigEndian(int index, byte[] value) { */ public void set(int index, int start, ArrowBuf buffer) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, buffer, start, TYPE_WIDTH); } /** @@ -264,10 +264,10 @@ public void setSafe(int index, int start, ArrowBuf buffer, int length) { // do the bound checks. buffer.checkBytes(start, start + length); - valueBuffer.checkBytes(index * TYPE_WIDTH, (index + 1) * TYPE_WIDTH); + valueBuffer.checkBytes((long) index * TYPE_WIDTH, (long) (index + 1) * TYPE_WIDTH); long inAddress = buffer.memoryAddress() + start; - long outAddress = valueBuffer.memoryAddress() + index * TYPE_WIDTH; + long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH; PlatformDependent.copyMemory(inAddress, outAddress, length); // sign extend if (length < 16) { @@ -291,11 +291,11 @@ public void setBigEndianSafe(int index, int start, ArrowBuf buffer, int length) // do the bound checks. buffer.checkBytes(start, start + length); - valueBuffer.checkBytes(index * TYPE_WIDTH, (index + 1) * TYPE_WIDTH); + valueBuffer.checkBytes((long) index * TYPE_WIDTH, (long) (index + 1) * TYPE_WIDTH); // not using buffer.getByte() to avoid boundary checks for every byte. long inAddress = buffer.memoryAddress() + start; - long outAddress = valueBuffer.memoryAddress() + index * TYPE_WIDTH; + long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH; // swap bytes to convert BE to LE for (int byteIdx = 0; byteIdx < length; ++byteIdx) { byte val = PlatformDependent.getByte((inAddress + length - 1) - byteIdx); @@ -345,7 +345,7 @@ public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentE throw new IllegalArgumentException(); } else if (holder.isSet > 0) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); } else { BitVectorHelper.unsetBit(validityBuffer, index); } @@ -359,7 +359,7 @@ public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentE */ public void set(int index, DecimalHolder holder) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java index 533d331de95..9c09347011a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java @@ -110,7 +110,7 @@ public MinorType getMinorType() { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } /** @@ -123,7 +123,7 @@ public ArrowBuf get(int index) throws IllegalStateException { if (isSet(index) == 0) { return null; } - return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + return valueBuffer.slice((long) index * TYPE_WIDTH, TYPE_WIDTH); } /** @@ -207,7 +207,7 @@ private StringBuilder getAsStringBuilderHelper(int index) { */ public void set(int index, ArrowBuf value) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, value, 0, TYPE_WIDTH); } /** @@ -217,7 +217,7 @@ public void set(int index, ArrowBuf value) { * @param value The duration value (in the timeunit associated with this vector) */ public void set(int index, long value) { - final int offsetIndex = index * TYPE_WIDTH; + final long offsetIndex = (long) index * TYPE_WIDTH; BitVectorHelper.setBit(validityBuffer, index); valueBuffer.setLong(offsetIndex, value); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java index aad185b58c5..630522b72bf 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java @@ -117,7 +117,7 @@ public byte[] get(int index) { return null; } final byte[] dst = new byte[byteWidth]; - valueBuffer.getBytes(index * byteWidth, dst, 0, byteWidth); + valueBuffer.getBytes((long) index * byteWidth, dst, 0, byteWidth); return dst; } @@ -136,7 +136,7 @@ public void get(int index, NullableFixedSizeBinaryHolder holder) { return; } holder.isSet = 1; - holder.buffer = valueBuffer.slice(index * byteWidth, byteWidth); + holder.buffer = valueBuffer.slice((long) index * byteWidth, byteWidth); } /** @@ -167,7 +167,7 @@ public void set(int index, byte[] value) { Preconditions.checkNotNull(value, "expecting a valid byte array"); assert byteWidth <= value.length; BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * byteWidth, value, 0, byteWidth); + valueBuffer.setBytes((long) index * byteWidth, value, 0, byteWidth); } /** @@ -205,7 +205,7 @@ public void set(int index, ArrowBuf buffer) { assert index >= 0; assert byteWidth <= buffer.capacity(); BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * byteWidth, buffer, 0, byteWidth); + valueBuffer.setBytes((long) index * byteWidth, buffer, 0, byteWidth); } /** @@ -316,7 +316,7 @@ public void setSafe(int index, NullableFixedSizeBinaryHolder holder) { */ public static byte[] get(final ArrowBuf buffer, final int index, final int byteWidth) { final byte[] dst = new byte[byteWidth]; - buffer.getBytes(index * byteWidth, dst, 0, byteWidth); + buffer.getBytes((long) index * byteWidth, dst, 0, byteWidth); return dst; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java index 74fbc32ca86..b083f3085fe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java @@ -114,7 +114,7 @@ public float get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getFloat(index * TYPE_WIDTH); + return valueBuffer.getFloat((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableFloat4Holder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getFloat(index * TYPE_WIDTH); + holder.value = valueBuffer.getFloat((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Float getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getFloat(index * TYPE_WIDTH); + return valueBuffer.getFloat((long) index * TYPE_WIDTH); } } @@ -155,7 +155,7 @@ public Float getObject(int index) { private void setValue(int index, float value) { - valueBuffer.setFloat(index * TYPE_WIDTH, value); + valueBuffer.setFloat((long) index * TYPE_WIDTH, value); } /** @@ -279,7 +279,7 @@ public void setSafe(int index, int isSet, float value) { * @return value stored at the index. */ public static float get(final ArrowBuf buffer, final int index) { - return buffer.getFloat(index * TYPE_WIDTH); + return buffer.getFloat((long) index * TYPE_WIDTH); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java index 9ca40616982..80da81f5fbe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java @@ -114,7 +114,7 @@ public double get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getDouble(index * TYPE_WIDTH); + return valueBuffer.getDouble((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableFloat8Holder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getDouble(index * TYPE_WIDTH); + holder.value = valueBuffer.getDouble((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Double getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getDouble(index * TYPE_WIDTH); + return valueBuffer.getDouble((long) index * TYPE_WIDTH); } } @@ -156,7 +156,7 @@ public Double getObject(int index) { private void setValue(int index, double value) { - valueBuffer.setDouble(index * TYPE_WIDTH, value); + valueBuffer.setDouble((long) index * TYPE_WIDTH, value); } /** @@ -280,7 +280,7 @@ public void setSafe(int index, int isSet, double value) { * @return value stored at the index. */ public static double get(final ArrowBuf buffer, final int index) { - return buffer.getDouble(index * TYPE_WIDTH); + return buffer.getDouble((long) index * TYPE_WIDTH); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java index e91e0eaa0e2..c2b9ee4b2d8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java @@ -114,7 +114,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableIntHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } } @@ -155,7 +155,7 @@ public Integer getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** @@ -279,7 +279,7 @@ public void setSafe(int index, int isSet, int value) { * @return value stored at the index. */ public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java index b67d697b989..c5f85a1c0f6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java @@ -118,7 +118,7 @@ public MinorType getMinorType() { * @return day value stored at the index. */ public static int getDays(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } /** @@ -132,7 +132,7 @@ public static int getDays(final ArrowBuf buffer, final int index) { * @return milliseconds value stored at the index. */ public static int getMilliseconds(final ArrowBuf buffer, final int index) { - return buffer.getInt((index * TYPE_WIDTH) + MILLISECOND_OFFSET); + return buffer.getInt((long) index * TYPE_WIDTH + MILLISECOND_OFFSET); } /** @@ -145,7 +145,7 @@ public ArrowBuf get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { return null; } - return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + return valueBuffer.slice((long) index * TYPE_WIDTH, TYPE_WIDTH); } /** @@ -160,7 +160,7 @@ public void get(int index, NullableIntervalDayHolder holder) { holder.isSet = 0; return; } - final int startIndex = index * TYPE_WIDTH; + final long startIndex = (long) index * TYPE_WIDTH; holder.isSet = 1; holder.days = valueBuffer.getInt(startIndex); holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); @@ -176,7 +176,7 @@ public Duration getObject(int index) { if (isSet(index) == 0) { return null; } else { - final int startIndex = index * TYPE_WIDTH; + final long startIndex = (long) index * TYPE_WIDTH; final int days = valueBuffer.getInt(startIndex); final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); return Duration.ofDays(days).plusMillis(milliseconds); @@ -199,7 +199,7 @@ public StringBuilder getAsStringBuilder(int index) { } private StringBuilder getAsStringBuilderHelper(int index) { - final int startIndex = index * TYPE_WIDTH; + final long startIndex = (long) index * TYPE_WIDTH; final int days = valueBuffer.getInt(startIndex); int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); @@ -238,7 +238,7 @@ private StringBuilder getAsStringBuilderHelper(int index) { */ public void set(int index, ArrowBuf value) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, value, 0, TYPE_WIDTH); } /** @@ -249,7 +249,7 @@ public void set(int index, ArrowBuf value) { * @param milliseconds milliseconds for the interval */ public void set(int index, int days, int milliseconds) { - final int offsetIndex = index * TYPE_WIDTH; + final long offsetIndex = (long) index * TYPE_WIDTH; BitVectorHelper.setBit(validityBuffer, index); valueBuffer.setInt(offsetIndex, days); valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java index 25c7535c005..4bb448fc037 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java @@ -117,7 +117,7 @@ public MinorType getMinorType() { * @return value stored at the index. */ public static int getTotalMonths(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -146,7 +146,7 @@ public void get(int index, NullableIntervalYearHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -159,7 +159,7 @@ public Period getObject(int index) { if (isSet(index) == 0) { return null; } else { - final int interval = valueBuffer.getInt(index * TYPE_WIDTH); + final int interval = valueBuffer.getInt((long) index * TYPE_WIDTH); // TODO: verify interval is in months return Period.ofMonths(interval); } @@ -181,7 +181,7 @@ public StringBuilder getAsStringBuilder(int index) { } private StringBuilder getAsStringBuilderHelper(int index) { - int value = valueBuffer.getInt(index * TYPE_WIDTH); + int value = valueBuffer.getInt((long) index * TYPE_WIDTH); final int years = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths); final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths); @@ -204,7 +204,7 @@ private StringBuilder getAsStringBuilderHelper(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java index 724daf59f15..b0453fc58db 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java @@ -114,7 +114,7 @@ public short get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getShort(index * TYPE_WIDTH); + return valueBuffer.getShort((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableSmallIntHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getShort(index * TYPE_WIDTH); + holder.value = valueBuffer.getShort((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Short getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getShort(index * TYPE_WIDTH); + return valueBuffer.getShort((long) index * TYPE_WIDTH); } } @@ -155,11 +155,11 @@ public Short getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setShort(index * TYPE_WIDTH, value); + valueBuffer.setShort((long) index * TYPE_WIDTH, value); } private void setValue(int index, short value) { - valueBuffer.setShort(index * TYPE_WIDTH, value); + valueBuffer.setShort((long) index * TYPE_WIDTH, value); } /** @@ -307,7 +307,7 @@ public void setSafe(int index, int isSet, short value) { * @return value stored at the index. */ public static short get(final ArrowBuf buffer, final int index) { - return buffer.getShort(index * TYPE_WIDTH); + return buffer.getShort((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java index b1ab36a6824..86adc2ad43d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java @@ -114,7 +114,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableTimeMicroHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } @@ -155,7 +155,7 @@ public Long getObject(int index) { private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** @@ -279,7 +279,7 @@ public void setSafe(int index, int isSet, long value) { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java index 8c7943dfd0d..93ebf1952ee 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java @@ -116,7 +116,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -132,7 +132,7 @@ public void get(int index, NullableTimeMilliHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -145,7 +145,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } - final int millis = valueBuffer.getInt(index * TYPE_WIDTH); + final int millis = valueBuffer.getInt((long) index * TYPE_WIDTH); // TODO: this doesn't seem right, time not from epoch return DateUtility.getLocalDateTimeFromEpochMilli(millis); } @@ -159,7 +159,7 @@ public LocalDateTime getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** @@ -284,7 +284,7 @@ public void setSafe(int index, int isSet, int value) { * @return value stored at the index. */ public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java index b711131e177..b296ba35772 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java @@ -114,7 +114,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableTimeNanoHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } @@ -156,7 +156,7 @@ public Long getObject(int index) { private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** @@ -280,7 +280,7 @@ public void setSafe(int index, int isSet, long value) { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java index 69e96b94084..eca4f304ef0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java @@ -114,7 +114,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableTimeSecHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } } @@ -156,7 +156,7 @@ public Integer getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** @@ -280,7 +280,7 @@ public void setSafe(int index, int isSet, int value) { * @return value stored at the index. */ public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java index 9c408e33610..ce34da99fe0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java @@ -120,7 +120,7 @@ public void get(int index, NullableTimeStampMicroTZHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -133,7 +133,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java index 0b822259f55..d9cb2b9c13a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java @@ -116,7 +116,7 @@ public void get(int index, NullableTimeStampMicroHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -129,7 +129,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long micros = valueBuffer.getLong(index * TYPE_WIDTH); + final long micros = valueBuffer.getLong((long) index * TYPE_WIDTH); return DateUtility.getLocalDateTimeFromEpochMicro(micros); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java index 47c59ac373e..b2870ba4f27 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java @@ -120,7 +120,7 @@ public void get(int index, NullableTimeStampMilliTZHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -133,7 +133,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java index 23c542e231e..55928154d5f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java @@ -116,7 +116,7 @@ public void get(int index, NullableTimeStampMilliHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -129,7 +129,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long millis = valueBuffer.getLong(index * TYPE_WIDTH); + final long millis = valueBuffer.getLong((long) index * TYPE_WIDTH); return DateUtility.getLocalDateTimeFromEpochMilli(millis); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java index b91f352e960..ff0987901a9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java @@ -120,7 +120,7 @@ public void get(int index, NullableTimeStampNanoTZHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -133,7 +133,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java index 4873b34bc94..5c2dd663997 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java @@ -116,7 +116,7 @@ public void get(int index, NullableTimeStampNanoHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -129,7 +129,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long nanos = valueBuffer.getLong(index * TYPE_WIDTH); + final long nanos = valueBuffer.getLong((long) index * TYPE_WIDTH); return DateUtility.getLocalDateTimeFromEpochNano(nanos); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java index 78bda17cfba..8034e9bf74d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java @@ -120,7 +120,7 @@ public void get(int index, NullableTimeStampSecTZHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -133,7 +133,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java index 7dfedf8c883..59660a25f03 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java @@ -116,7 +116,7 @@ public void get(int index, NullableTimeStampSecHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -129,7 +129,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long secs = valueBuffer.getLong(index * TYPE_WIDTH); + final long secs = valueBuffer.getLong((long) index * TYPE_WIDTH); final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs); return DateUtility.getLocalDateTimeFromEpochMilli(millis); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java index 271fdb5ee0c..b2202994306 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java @@ -74,7 +74,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } @@ -86,7 +86,7 @@ public long get(int index) throws IllegalStateException { protected void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** @@ -154,7 +154,7 @@ public void setSafe(int index, int isSet, long value) { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java index 349a50a69a6..f2b834d9382 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java @@ -80,7 +80,7 @@ public MinorType getMinorType() { * @return value stored at the index. */ public static char get(final ArrowBuf buffer, final int index) { - return buffer.getChar(index * TYPE_WIDTH); + return buffer.getChar((long) index * TYPE_WIDTH); } /** @@ -93,7 +93,7 @@ public char get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getChar(index * TYPE_WIDTH); + return valueBuffer.getChar((long) index * TYPE_WIDTH); } /** @@ -109,7 +109,7 @@ public void get(int index, NullableUInt2Holder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getChar(index * TYPE_WIDTH); + holder.value = valueBuffer.getChar((long) index * TYPE_WIDTH); } /** @@ -122,7 +122,7 @@ public Character getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getChar(index * TYPE_WIDTH); + return valueBuffer.getChar((long) index * TYPE_WIDTH); } } @@ -135,11 +135,11 @@ public Character getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setChar(index * TYPE_WIDTH, value); + valueBuffer.setChar((long) index * TYPE_WIDTH, value); } private void setValue(int index, char value) { - valueBuffer.setChar(index * TYPE_WIDTH, value); + valueBuffer.setChar((long) index * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java index a747026a176..0c99b7dcf08 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java @@ -83,7 +83,7 @@ public MinorType getMinorType() { * @return value stored at the index. */ public static long getNoOverflow(final ArrowBuf buffer, final int index) { - long l = buffer.getInt(index * TYPE_WIDTH); + long l = buffer.getInt((long) index * TYPE_WIDTH); return (0x00000000FFFFFFFFL) & l; } @@ -97,7 +97,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -113,7 +113,7 @@ public void get(int index, NullableUInt4Holder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -126,7 +126,7 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } } @@ -153,7 +153,7 @@ public Long getObjectNoOverflow(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java index 19d761b7857..2660e873a9f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java @@ -87,7 +87,7 @@ public MinorType getMinorType() { * @return value stored at the index. */ public static BigInteger getNoOverflow(final ArrowBuf buffer, final int index) { - BigInteger l = BigInteger.valueOf(buffer.getLong(index * TYPE_WIDTH)); + BigInteger l = BigInteger.valueOf(buffer.getLong((long) index * TYPE_WIDTH)); return SAFE_CONVERSION_MASK.and(l); } @@ -102,7 +102,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -118,7 +118,7 @@ public void get(int index, NullableUInt8Holder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -131,7 +131,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } @@ -158,7 +158,7 @@ public BigInteger getObjectNoOverflow(int index) { private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java index 2703d261219..83194cbc22e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java @@ -73,7 +73,7 @@ public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int sca */ public static byte[] getByteArrayFromArrowBuf(ArrowBuf bytebuf, int index) { final byte[] value = new byte[DECIMAL_BYTE_LENGTH]; - final int startIndex = index * DECIMAL_BYTE_LENGTH; + final long startIndex = (long) index * DECIMAL_BYTE_LENGTH; bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH); return value; } @@ -127,7 +127,7 @@ public static void writeBigDecimalToArrowBuf(BigDecimal value, ArrowBuf bytebuf, * Write the given long to the ArrowBuf at the given value index. */ public static void writeLongToArrowBuf(long value, ArrowBuf bytebuf, int index) { - final long addressOfValue = bytebuf.memoryAddress() + index * DECIMAL_BYTE_LENGTH; + final long addressOfValue = bytebuf.memoryAddress() + (long) index * DECIMAL_BYTE_LENGTH; PlatformDependent.putLong(addressOfValue, value); final long padValue = Long.signum(value) == -1 ? -1L : 0L; PlatformDependent.putLong(addressOfValue + Long.BYTES, padValue); @@ -143,7 +143,7 @@ public static void writeByteArrayToArrowBuf(byte[] bytes, ArrowBuf bytebuf, int } private static void writeByteArrayToArrowBufHelper(byte[] bytes, ArrowBuf bytebuf, int index) { - final int startIndex = index * DECIMAL_BYTE_LENGTH; + final long startIndex = (long) index * DECIMAL_BYTE_LENGTH; if (bytes.length > DECIMAL_BYTE_LENGTH) { throw new UnsupportedOperationException("Decimal size greater than 16 bytes"); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java new file mode 100644 index 00000000000..dad871e060a --- /dev/null +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.DefaultAllocationManagerOption; +import org.apache.arrow.memory.RootAllocator; + +import io.netty.buffer.ArrowBuf; + +/** + * Integration test for a vector with a large (more than 2GB) {@link io.netty.buffer.ArrowBuf} as + * the data buffer. + * To run this test, please + *
  • Make sure there are 4GB memory available in the system.
    • + *
  • + * Make sure the default allocation manager type is unsafe. + * This can be achieved by the environmental variable or system property. + * The details can be found in {@link DefaultAllocationManagerOption}. + *
    • + */ +public class TestLargeVector { + private static void testLargeLongVector() { + System.out.println("Testing large big int vector."); + + final long bufSize = 4 * 1024 * 1024 * 1024L; + final int vecLength = (int) (bufSize / BigIntVector.TYPE_WIDTH); + + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + BigIntVector largeVec = new BigIntVector("vec", allocator)) { + largeVec.allocateNew(vecLength); + + System.out.println("Successfully allocated a vector with capacity " + vecLength); + + for (int i = 0; i < vecLength; i++) { + largeVec.set(i, i * 10L); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully written " + (i + 1) + " values"); + } + } + System.out.println("Successfully written " + vecLength + " values"); + + for (int i = 0; i < vecLength; i++) { + long val = largeVec.get(i); + assertEquals(i * 10L, val); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully read " + (i + 1) + " values"); + } + } + System.out.println("Successfully read " + vecLength + " values"); + } + System.out.println("Successfully released the large vector."); + } + + private static void testLargeIntVector() { + System.out.println("Testing large int vector."); + + final long bufSize = 4 * 1024 * 1024 * 1024L; + final int vecLength = (int) (bufSize / IntVector.TYPE_WIDTH); + + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + IntVector largeVec = new IntVector("vec", allocator)) { + largeVec.allocateNew(vecLength); + + System.out.println("Successfully allocated a vector with capacity " + vecLength); + + for (int i = 0; i < vecLength; i++) { + largeVec.set(i, i); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully written " + (i + 1) + " values"); + } + } + System.out.println("Successfully written " + vecLength + " values"); + + for (int i = 0; i < vecLength; i++) { + long val = largeVec.get(i); + assertEquals(i, val); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully read " + (i + 1) + " values"); + } + } + System.out.println("Successfully read " + vecLength + " values"); + } + System.out.println("Successfully released the large vector."); + } + + private static void testLargeDecimalVector() { + System.out.println("Testing large decimal vector."); + + final long bufSize = 4 * 1024 * 1024 * 1024L; + final int vecLength = (int) (bufSize / DecimalVector.TYPE_WIDTH); + + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + DecimalVector largeVec = new DecimalVector("vec", allocator, 38, 16)) { + largeVec.allocateNew(vecLength); + + System.out.println("Successfully allocated a vector with capacity " + vecLength); + + for (int i = 0; i < vecLength; i++) { + largeVec.set(i, 0); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully written " + (i + 1) + " values"); + } + } + System.out.println("Successfully written " + vecLength + " values"); + + for (int i = 0; i < vecLength; i++) { + ArrowBuf buf = largeVec.get(i); + assertEquals(buf.capacity(), DecimalVector.TYPE_WIDTH); + assertEquals(0, buf.getLong(0)); + assertEquals(0, buf.getLong(8)); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully read " + (i + 1) + " values"); + } + } + System.out.println("Successfully read " + vecLength + " values"); + } + System.out.println("Successfully released the large vector."); + } + + private static void testLargeFixedSizeBinaryVector() { + System.out.println("Testing large fixed size binary vector."); + + final long bufSize = 4 * 1024 * 1024 * 1024L; + final int typeWidth = 8; + final int vecLength = (int) (bufSize / typeWidth); + + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + FixedSizeBinaryVector largeVec = new FixedSizeBinaryVector("vec", allocator, typeWidth)) { + largeVec.allocateNew(vecLength); + + System.out.println("Successfully allocated a vector with capacity " + vecLength); + + byte[] value = new byte[] {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}; + for (int i = 0; i < vecLength; i++) { + largeVec.set(i, value); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully written " + (i + 1) + " values"); + } + } + System.out.println("Successfully written " + vecLength + " values"); + + for (int i = 0; i < vecLength; i++) { + byte[] buf = largeVec.get(i); + assertEquals(typeWidth, buf.length); + assertArrayEquals(buf, value); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully read " + (i + 1) + " values"); + } + } + System.out.println("Successfully read " + vecLength + " values"); + } + System.out.println("Successfully released the large vector."); + } + + public static void main(String[] args) { + testLargeLongVector(); + testLargeIntVector(); + testLargeDecimalVector(); + testLargeFixedSizeBinaryVector(); + } +} From 6e2cafe84a9969accd4acc4b9830f9a7484dd431 Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Thu, 27 Feb 2020 18:00:02 +0800 Subject: [PATCH 2/8] [ARROW-7610][Java] Use the unsafe allocator by default --- .../org/apache/arrow/memory/DefaultAllocationManagerOption.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java index 46a54734889..d631c15278f 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java @@ -90,7 +90,7 @@ private static AllocationManager.Factory getDefaultAllocationManagerFactory() { return UnsafeAllocationManager.FACTORY; case Unknown: logger.warn("allocation manager type not specified, using netty as the default type"); - return NettyAllocationManager.FACTORY; + return UnsafeAllocationManager.FACTORY; default: throw new IllegalStateException("Unknown allocation manager type: " + type); } From da1a06abf6b7ff0108bb4934691e492906319831 Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Mon, 6 Apr 2020 12:29:14 +0800 Subject: [PATCH 3/8] [ARROW-7610][Java] Resolve comments --- .../DefaultAllocationManagerOption.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java index d631c15278f..c4c27440992 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java @@ -63,19 +63,19 @@ public enum AllocationManagerType { private static AllocationManagerType getDefaultAllocationManagerType() { AllocationManagerType ret = AllocationManagerType.Unknown; - String envValue = System.getenv(ALLOCATION_MANAGER_TYPE_ENV_NAME); - if ("netty".equals(envValue)) { - ret = AllocationManagerType.Netty; - } else if ("unsafe".equals(envValue)) { - ret = AllocationManagerType.Unsafe; + try { + String envValue = System.getenv(ALLOCATION_MANAGER_TYPE_ENV_NAME); + ret = AllocationManagerType.valueOf(envValue); + } catch (IllegalArgumentException | NullPointerException e) { + // ignore the exception, and make the allocation manager type remain unchanged } // system property takes precedence - String propValue = System.getProperty(ALLOCATION_MANAGER_TYPE_PROPERTY_NAME); - if ("netty".equals(propValue)) { - ret = AllocationManagerType.Netty; - } else if ("unsafe".equals(propValue)) { - ret = AllocationManagerType.Unsafe; + try { + String propValue = System.getProperty(ALLOCATION_MANAGER_TYPE_PROPERTY_NAME); + ret = AllocationManagerType.valueOf(propValue); + } catch (IllegalArgumentException | NullPointerException e) { + // ignore the exception, and make the allocation manager type remain unchanged } return ret; } @@ -89,8 +89,8 @@ private static AllocationManager.Factory getDefaultAllocationManagerFactory() { case Unsafe: return UnsafeAllocationManager.FACTORY; case Unknown: - logger.warn("allocation manager type not specified, using netty as the default type"); - return UnsafeAllocationManager.FACTORY; + logger.info("allocation manager type not specified, using netty as the default type"); + return NettyAllocationManager.FACTORY; default: throw new IllegalStateException("Unknown allocation manager type: " + type); } From 78b77a850c107d5993956286a6ba6ad6d449e41c Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Tue, 7 Apr 2020 17:08:35 +0800 Subject: [PATCH 4/8] [ARROW-7610][Java] Resolve log related problems --- .../apache/arrow/memory/DefaultAllocationManagerOption.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java index c4c27440992..1f3ad036fe4 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java @@ -32,7 +32,7 @@ public class DefaultAllocationManagerOption { */ public static final String ALLOCATION_MANAGER_TYPE_PROPERTY_NAME = "arrow.allocation.manager.type"; - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DefaultAllocationManagerOption.class); + static final org.slf4j.Logger LOGGER = org.slf4j.LoggerFactory.getLogger(DefaultAllocationManagerOption.class); /** * The default allocation manager factory. @@ -89,7 +89,7 @@ private static AllocationManager.Factory getDefaultAllocationManagerFactory() { case Unsafe: return UnsafeAllocationManager.FACTORY; case Unknown: - logger.info("allocation manager type not specified, using netty as the default type"); + LOGGER.info("allocation manager type not specified, using netty as the default type"); return NettyAllocationManager.FACTORY; default: throw new IllegalStateException("Unknown allocation manager type: " + type); From 6c0edf628345822c180395bdd969e94296b4bbc9 Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Thu, 16 Apr 2020 17:20:45 +0800 Subject: [PATCH 5/8] [ARROW-7610][Java] Revise netty allocator to support large buffers --- .../apache/arrow/memory/BaseAllocator.java | 2 +- .../org/apache/arrow/memory/BufferLedger.java | 7 -- .../DefaultAllocationManagerOption.java | 98 ------------------- .../arrow/memory/NettyAllocationManager.java | 28 ++---- .../arrow/memory/UnsafeAllocationManager.java | 65 ------------ .../apache/arrow/vector/TestLargeVector.java | 1 - 6 files changed, 11 insertions(+), 190 deletions(-) delete mode 100644 java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java delete mode 100644 java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java index 0346739fc21..84a8f63e591 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java @@ -765,7 +765,7 @@ abstract static class Config { */ @Value.Default AllocationManager.Factory getAllocationManagerFactory() { - return DefaultAllocationManagerOption.DEFAULT_ALLOCATION_MANAGER_FACTORY; + return NettyAllocationManager.FACTORY; } /** diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java index d1091f1f471..c133aa1065b 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java @@ -554,13 +554,6 @@ public T unwrap(Class clazz) { return clazz.cast(allocationManager); } - // TODO: remove this in a future release. - if (clazz == UnsafeDirectLittleEndian.class) { - Preconditions.checkState(allocationManager instanceof NettyAllocationManager, - "Underlying memory was not allocated by Netty"); - return clazz.cast(((NettyAllocationManager) allocationManager).getMemoryChunk()); - } - throw new IllegalArgumentException("Unexpected unwrapping class: " + clazz); } } diff --git a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java deleted file mode 100644 index 1f3ad036fe4..00000000000 --- a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.memory; - -/** - * A class for choosing the default allocation manager. - */ -public class DefaultAllocationManagerOption { - - /** - * The environmental variable to set the default allocation manager type. - */ - public static final String ALLOCATION_MANAGER_TYPE_ENV_NAME = "ARROW_ALLOCATION_MANAGER_TYPE"; - - /** - * The system property to set the default allocation manager type. - */ - public static final String ALLOCATION_MANAGER_TYPE_PROPERTY_NAME = "arrow.allocation.manager.type"; - - static final org.slf4j.Logger LOGGER = org.slf4j.LoggerFactory.getLogger(DefaultAllocationManagerOption.class); - - /** - * The default allocation manager factory. - */ - public static final AllocationManager.Factory DEFAULT_ALLOCATION_MANAGER_FACTORY = - getDefaultAllocationManagerFactory(); - - /** - * The allocation manager type. - */ - public enum AllocationManagerType { - /** - * Netty based allocation manager. - */ - Netty, - - /** - * Unsafe based allocation manager. - */ - Unsafe, - - /** - * Unknown type. - */ - Unknown, - } - - private static AllocationManagerType getDefaultAllocationManagerType() { - AllocationManagerType ret = AllocationManagerType.Unknown; - - try { - String envValue = System.getenv(ALLOCATION_MANAGER_TYPE_ENV_NAME); - ret = AllocationManagerType.valueOf(envValue); - } catch (IllegalArgumentException | NullPointerException e) { - // ignore the exception, and make the allocation manager type remain unchanged - } - - // system property takes precedence - try { - String propValue = System.getProperty(ALLOCATION_MANAGER_TYPE_PROPERTY_NAME); - ret = AllocationManagerType.valueOf(propValue); - } catch (IllegalArgumentException | NullPointerException e) { - // ignore the exception, and make the allocation manager type remain unchanged - } - return ret; - } - - private static AllocationManager.Factory getDefaultAllocationManagerFactory() { - AllocationManagerType type = getDefaultAllocationManagerType(); - - switch (type) { - case Netty: - return NettyAllocationManager.FACTORY; - case Unsafe: - return UnsafeAllocationManager.FACTORY; - case Unknown: - LOGGER.info("allocation manager type not specified, using netty as the default type"); - return NettyAllocationManager.FACTORY; - default: - throw new IllegalStateException("Unknown allocation manager type: " + type); - } - } -} diff --git a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java index e6da8a2a8aa..d31855b38d4 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java @@ -17,10 +17,9 @@ package org.apache.arrow.memory; -import org.apache.arrow.memory.util.LargeMemoryUtil; - import io.netty.buffer.PooledByteBufAllocatorL; import io.netty.buffer.UnsafeDirectLittleEndian; +import io.netty.util.internal.PlatformDependent; /** * The default implementation of {@link AllocationManager}. The implementation is responsible for managing when memory @@ -34,31 +33,24 @@ public class NettyAllocationManager extends AllocationManager { static final UnsafeDirectLittleEndian EMPTY = INNER_ALLOCATOR.empty; static final long CHUNK_SIZE = INNER_ALLOCATOR.getChunkSize(); - private final int allocatedSize; - private final UnsafeDirectLittleEndian memoryChunk; + private final long allocatedSize; - NettyAllocationManager(BaseAllocator accountingAllocator, int requestedSize) { - super(accountingAllocator); - this.memoryChunk = INNER_ALLOCATOR.allocate(requestedSize); - this.allocatedSize = memoryChunk.capacity(); - } + private final long allocatedAddress; - /** - * Get the underlying memory chunk managed by this AllocationManager. - * @return buffer - */ - UnsafeDirectLittleEndian getMemoryChunk() { - return memoryChunk; + NettyAllocationManager(BaseAllocator accountingAllocator, long requestedSize) { + super(accountingAllocator); + allocatedAddress = PlatformDependent.allocateMemory(requestedSize); + allocatedSize = requestedSize; } @Override protected long memoryAddress() { - return memoryChunk.memoryAddress(); + return allocatedAddress; } @Override protected void release0() { - memoryChunk.release(); + PlatformDependent.freeMemory(allocatedAddress); } /** @@ -79,7 +71,7 @@ private Factory() {} @Override public AllocationManager create(BaseAllocator accountingAllocator, long size) { - return new NettyAllocationManager(accountingAllocator, LargeMemoryUtil.checkedCastToInt(size)); + return new NettyAllocationManager(accountingAllocator, size); } } } diff --git a/java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java deleted file mode 100644 index 8d67919f7f1..00000000000 --- a/java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.memory; - -import io.netty.util.internal.PlatformDependent; - -/** - * Allocation manager based on unsafe API. - */ -public class UnsafeAllocationManager extends AllocationManager { - - public static final Factory FACTORY = new Factory(); - - private final long allocatedSize; - - private final long allocatedAddress; - - protected UnsafeAllocationManager(BaseAllocator accountingAllocator, long requestedSize) { - super(accountingAllocator); - allocatedAddress = PlatformDependent.allocateMemory(requestedSize); - allocatedSize = requestedSize; - } - - @Override - public long getSize() { - return allocatedSize; - } - - @Override - protected long memoryAddress() { - return allocatedAddress; - } - - @Override - protected void release0() { - PlatformDependent.freeMemory(allocatedAddress); - } - - /** - * Factory for creating {@link UnsafeAllocationManager}. - */ - public static class Factory implements AllocationManager.Factory { - private Factory() {} - - @Override - public AllocationManager create(BaseAllocator accountingAllocator, long size) { - return new UnsafeAllocationManager(accountingAllocator, size); - } - } -} diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java index dad871e060a..82326aa8d9e 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java @@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.DefaultAllocationManagerOption; import org.apache.arrow.memory.RootAllocator; import io.netty.buffer.ArrowBuf; From 78ca3dbb702f6149e85f57985a2c65ef5c3abfd8 Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Tue, 21 Apr 2020 18:30:13 +0800 Subject: [PATCH 6/8] [ARROW-7610][Java] Make netty allocation manager switch allocation strategy --- .../arrow/memory/NettyAllocationManager.java | 16 +++++++++++++--- .../apache/arrow/memory/TestLargeArrowBuf.java | 14 ++++---------- .../org/apache/arrow/vector/TestLargeVector.java | 8 +------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java index d31855b38d4..5c397c7cfd9 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java @@ -34,12 +34,18 @@ public class NettyAllocationManager extends AllocationManager { static final long CHUNK_SIZE = INNER_ALLOCATOR.getChunkSize(); private final long allocatedSize; - + private final UnsafeDirectLittleEndian memoryChunk; private final long allocatedAddress; NettyAllocationManager(BaseAllocator accountingAllocator, long requestedSize) { super(accountingAllocator); - allocatedAddress = PlatformDependent.allocateMemory(requestedSize); + if (requestedSize > Integer.MAX_VALUE) { + memoryChunk = null; + allocatedAddress = PlatformDependent.allocateMemory(requestedSize); + } else { + this.memoryChunk = INNER_ALLOCATOR.allocate(requestedSize); + allocatedAddress = memoryChunk.memoryAddress(); + } allocatedSize = requestedSize; } @@ -50,7 +56,11 @@ protected long memoryAddress() { @Override protected void release0() { - PlatformDependent.freeMemory(allocatedAddress); + if (allocatedSize > Integer.MAX_VALUE) { + PlatformDependent.freeMemory(allocatedAddress); + } else { + memoryChunk.release(); + } } /** diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java b/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java index 1448fcae122..d1e8d3b9b20 100644 --- a/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java +++ b/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java @@ -23,18 +23,11 @@ /** * Integration test for large (more than 2GB) {@link io.netty.buffer.ArrowBuf}. - * To run this test, please - *
  • Make sure there are 4GB memory available in the system.
    • - *
  • - * Make sure the default allocation manager type is unsafe. - * This can be achieved by the environmental variable or system property. - * The details can be found in {@link DefaultAllocationManagerOption}. - *
    • + * To run this test, please make sure there is at least 4GB memory in the system. */ public class TestLargeArrowBuf { - private static void testLargeArrowBuf() { - final long bufSize = 4 * 1024 * 1024 * 1024L; + private static void testLargeArrowBuf(long bufSize) { try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); ArrowBuf largeBuf = allocator.buffer(bufSize)) { assertEquals(bufSize, largeBuf.capacity()); @@ -63,6 +56,7 @@ private static void testLargeArrowBuf() { } public static void main(String[] args) { - testLargeArrowBuf(); + testLargeArrowBuf(4 * 1024 * 1024 * 1024L); + testLargeArrowBuf(Integer.MAX_VALUE); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java index 82326aa8d9e..4d7664731e0 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java @@ -28,13 +28,7 @@ /** * Integration test for a vector with a large (more than 2GB) {@link io.netty.buffer.ArrowBuf} as * the data buffer. - * To run this test, please - *
  • Make sure there are 4GB memory available in the system.
    • - *
  • - * Make sure the default allocation manager type is unsafe. - * This can be achieved by the environmental variable or system property. - * The details can be found in {@link DefaultAllocationManagerOption}. - *
    • + * To run this test, please make sure there is at least 4GB free memory in the system. */ public class TestLargeVector { private static void testLargeLongVector() { From a8ab2305320cd9d1dcfa30709932993c0d0175b6 Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Fri, 24 Apr 2020 14:34:46 +0800 Subject: [PATCH 7/8] [ARROW-7610][Java] Make allocation cut-off value configurable --- .../arrow/memory/NettyAllocationManager.java | 61 ++++++++++-- .../memory/TestNettyAllocationManager.java | 98 +++++++++++++++++++ java/pom.xml | 1 + 3 files changed, 153 insertions(+), 7 deletions(-) create mode 100644 java/memory/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java index 5c397c7cfd9..ce9b996ebb2 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java @@ -29,6 +29,26 @@ public class NettyAllocationManager extends AllocationManager { public static final Factory FACTORY = new Factory(); + /** + * The default cut-off value for switching allocation strategies. + * If the request size is not greater than the cut-off value, we will allocate memory by + * {@link PooledByteBufAllocatorL} APIs, + * otherwise, we will use {@link PlatformDependent} APIs. + */ + public static final long DEFAULT_ALLOCATION_CUTOFF_VALUE; + + public static final String DEFAULT_ALLOCATION_CUTOFF_NAME = "default.allocation.cutoff.name"; + + static { + long cutOffValue; + try { + cutOffValue = Long.parseLong(System.getProperty(DEFAULT_ALLOCATION_CUTOFF_NAME)); + } catch (Exception e) { + cutOffValue = Integer.MAX_VALUE; + } + DEFAULT_ALLOCATION_CUTOFF_VALUE = cutOffValue; + } + private static final PooledByteBufAllocatorL INNER_ALLOCATOR = new PooledByteBufAllocatorL(); static final UnsafeDirectLittleEndian EMPTY = INNER_ALLOCATOR.empty; static final long CHUNK_SIZE = INNER_ALLOCATOR.getChunkSize(); @@ -37,16 +57,43 @@ public class NettyAllocationManager extends AllocationManager { private final UnsafeDirectLittleEndian memoryChunk; private final long allocatedAddress; - NettyAllocationManager(BaseAllocator accountingAllocator, long requestedSize) { + /** + * The cut-off value for switching allocation strategies. + */ + private final long allocationCutOffValue; + + NettyAllocationManager(BaseAllocator accountingAllocator, long requestedSize, long allocationCutOffValue) { super(accountingAllocator); - if (requestedSize > Integer.MAX_VALUE) { - memoryChunk = null; - allocatedAddress = PlatformDependent.allocateMemory(requestedSize); + if (allocationCutOffValue > Integer.MAX_VALUE) { + throw new IllegalArgumentException("The cut-off value cannot be larger than Integer.MAX_VALUE"); + } + this.allocationCutOffValue = allocationCutOffValue; + + if (requestedSize > allocationCutOffValue) { + this.memoryChunk = null; + this.allocatedAddress = PlatformDependent.allocateMemory(requestedSize); + this.allocatedSize = requestedSize; } else { this.memoryChunk = INNER_ALLOCATOR.allocate(requestedSize); - allocatedAddress = memoryChunk.memoryAddress(); + this.allocatedAddress = memoryChunk.memoryAddress(); + this.allocatedSize = memoryChunk.capacity(); } - allocatedSize = requestedSize; + } + + NettyAllocationManager(BaseAllocator accountingAllocator, long requestedSize) { + this(accountingAllocator, requestedSize, DEFAULT_ALLOCATION_CUTOFF_VALUE); + } + + /** + * Get the underlying memory chunk managed by this AllocationManager. + * @return the underlying memory chunk if the request size is not greater than the + * {@link NettyAllocationManager#allocationCutOffValue}, or null otherwise. + * + * @deprecated this method will be removed in a future release. + */ + @Deprecated + UnsafeDirectLittleEndian getMemoryChunk() { + return allocatedSize > allocationCutOffValue ? null : memoryChunk; } @Override @@ -56,7 +103,7 @@ protected long memoryAddress() { @Override protected void release0() { - if (allocatedSize > Integer.MAX_VALUE) { + if (allocatedSize > allocationCutOffValue) { PlatformDependent.freeMemory(allocatedAddress); } else { memoryChunk.release(); diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java b/java/memory/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java new file mode 100644 index 00000000000..74d016fba8d --- /dev/null +++ b/java/memory/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import io.netty.buffer.ArrowBuf; + +/** + * Test cases for {@link NettyAllocationManager}. + */ +public class TestNettyAllocationManager { + + private void readWriteArrowBuf(ArrowBuf buffer) { + // write buffer + for (long i = 0; i < buffer.capacity() / 8; i++) { + buffer.setLong(i * 8, i); + } + + // read buffer + for (long i = 0; i < buffer.capacity() / 8; i++) { + long val = buffer.getLong(i * 8); + assertEquals(i, val); + } + } + + /** + * Test the allocation strategy for small buffers.. + */ + @Test + public void testSmallBufferAllocation() { + final long bufSize = 512L; + try (RootAllocator allocator = new RootAllocator(bufSize); + ArrowBuf buffer = allocator.buffer(bufSize)) { + // make sure the buffer is small enough, so we wil use the allocation strategy for small buffers + assertTrue(bufSize < NettyAllocationManager.DEFAULT_ALLOCATION_CUTOFF_VALUE); + + assertTrue(buffer.getReferenceManager() instanceof BufferLedger); + BufferLedger bufferLedger = (BufferLedger) buffer.getReferenceManager(); + + // make sure we are using netty allocation manager + AllocationManager allocMgr = bufferLedger.getAllocationManager(); + assertTrue(allocMgr instanceof NettyAllocationManager); + NettyAllocationManager nettyMgr = (NettyAllocationManager) allocMgr; + + // for the small buffer allocation strategy, the chunk is not null + assertNotNull(nettyMgr.getMemoryChunk()); + + readWriteArrowBuf(buffer); + } + } + + /** + * Test the allocation strategy for large buffers.. + */ + @Test + public void testLargeBufferAllocation() { + final long bufSize = 2048L; + try (RootAllocator allocator = new RootAllocator(bufSize); + ArrowBuf buffer = allocator.buffer(bufSize)) { + // make sure the buffer is large enough, so we wil use the allocation strategy for large buffers + assertTrue(bufSize > NettyAllocationManager.DEFAULT_ALLOCATION_CUTOFF_VALUE); + + assertTrue(buffer.getReferenceManager() instanceof BufferLedger); + BufferLedger bufferLedger = (BufferLedger) buffer.getReferenceManager(); + + // make sure we are using netty allocation manager + AllocationManager allocMgr = bufferLedger.getAllocationManager(); + assertTrue(allocMgr instanceof NettyAllocationManager); + NettyAllocationManager nettyMgr = (NettyAllocationManager) allocMgr; + + // for the large buffer allocation strategy, the chunk is null + assertNull(nettyMgr.getMemoryChunk()); + + readWriteArrowBuf(buffer); + } + } +} diff --git a/java/pom.xml b/java/pom.xml index acbfdd449b9..4165c3bed50 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -404,6 +404,7 @@ ${project.build.directory} true 1048576 + 1024 UTC