diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java index d1091f1f471..c133aa1065b 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java @@ -554,13 +554,6 @@ public T unwrap(Class clazz) { return clazz.cast(allocationManager); } - // TODO: remove this in a future release. - if (clazz == UnsafeDirectLittleEndian.class) { - Preconditions.checkState(allocationManager instanceof NettyAllocationManager, - "Underlying memory was not allocated by Netty"); - return clazz.cast(((NettyAllocationManager) allocationManager).getMemoryChunk()); - } - throw new IllegalArgumentException("Unexpected unwrapping class: " + clazz); } } diff --git a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java index 35b02f54ecd..341401708c7 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java @@ -17,48 +17,82 @@ package org.apache.arrow.memory; -import org.apache.arrow.memory.util.LargeMemoryUtil; - import io.netty.buffer.PooledByteBufAllocatorL; import io.netty.buffer.UnsafeDirectLittleEndian; +import io.netty.util.internal.PlatformDependent; /** - * The default implementation of AllocationManagerBase. The implementation is responsible for managing when memory + * The default implementation of {@link AllocationManager}. The implementation is responsible for managing when memory * is allocated and returned to the Netty-based PooledByteBufAllocatorL. */ public class NettyAllocationManager extends AllocationManager { public static final Factory FACTORY = new Factory(); + /** + * The default cut-off value for switching allocation strategies. + * If the request size is not greater than the cut-off value, we will allocate memory by + * {@link PooledByteBufAllocatorL} APIs, + * otherwise, we will use {@link PlatformDependent} APIs. + */ + public static final int DEFAULT_ALLOCATION_CUTOFF_VALUE = Integer.MAX_VALUE; + private static final PooledByteBufAllocatorL INNER_ALLOCATOR = new PooledByteBufAllocatorL(); static final UnsafeDirectLittleEndian EMPTY = INNER_ALLOCATOR.empty; static final long CHUNK_SIZE = INNER_ALLOCATOR.getChunkSize(); - private final int allocatedSize; + private final long allocatedSize; private final UnsafeDirectLittleEndian memoryChunk; + private final long allocatedAddress; + + /** + * The cut-off value for switching allocation strategies. + */ + private final int allocationCutOffValue; - NettyAllocationManager(BaseAllocator accountingAllocator, int requestedSize) { + NettyAllocationManager(BaseAllocator accountingAllocator, long requestedSize, int allocationCutOffValue) { super(accountingAllocator); - this.memoryChunk = INNER_ALLOCATOR.allocate(requestedSize); - this.allocatedSize = memoryChunk.capacity(); + this.allocationCutOffValue = allocationCutOffValue; + + if (requestedSize > allocationCutOffValue) { + this.memoryChunk = null; + this.allocatedAddress = PlatformDependent.allocateMemory(requestedSize); + this.allocatedSize = requestedSize; + } else { + this.memoryChunk = INNER_ALLOCATOR.allocate(requestedSize); + this.allocatedAddress = memoryChunk.memoryAddress(); + this.allocatedSize = memoryChunk.capacity(); + } + } + + NettyAllocationManager(BaseAllocator accountingAllocator, long requestedSize) { + this(accountingAllocator, requestedSize, DEFAULT_ALLOCATION_CUTOFF_VALUE); } /** * Get the underlying memory chunk managed by this AllocationManager. - * @return buffer + * @return the underlying memory chunk if the request size is not greater than the + * {@link NettyAllocationManager#allocationCutOffValue}, or null otherwise. + * + * @deprecated this method will be removed in a future release. */ + @Deprecated UnsafeDirectLittleEndian getMemoryChunk() { return memoryChunk; } @Override protected long memoryAddress() { - return memoryChunk.memoryAddress(); + return allocatedAddress; } @Override protected void release0() { - memoryChunk.release(); + if (memoryChunk == null) { + PlatformDependent.freeMemory(allocatedAddress); + } else { + memoryChunk.release(); + } } /** @@ -79,7 +113,7 @@ private Factory() {} @Override public AllocationManager create(BaseAllocator accountingAllocator, long size) { - return new NettyAllocationManager(accountingAllocator, LargeMemoryUtil.checkedCastToInt(size)); + return new NettyAllocationManager(accountingAllocator, size); } } } diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java b/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java new file mode 100644 index 00000000000..2160930f0a1 --- /dev/null +++ b/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import static org.junit.Assert.assertEquals; + +import io.netty.buffer.ArrowBuf; + +/** + * Integration test for large (more than 2GB) {@link io.netty.buffer.ArrowBuf}. + * To run this test, please make sure there is at least 4GB memory in the system. + *

+ * Please note that this is not a standard test case, so please run it by manually invoking the + * main method. + *

+ */ +public class TestLargeArrowBuf { + + private static void testLargeArrowBuf(long bufSize) { + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + ArrowBuf largeBuf = allocator.buffer(bufSize)) { + assertEquals(bufSize, largeBuf.capacity()); + System.out.println("Successfully allocated a buffer with capacity " + largeBuf.capacity()); + + for (long i = 0; i < bufSize / 8; i++) { + largeBuf.setLong(i * 8, i); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully written " + (i + 1) + " long words"); + } + } + System.out.println("Successfully written " + (bufSize / 8) + " long words"); + + for (long i = 0; i < bufSize / 8; i++) { + long val = largeBuf.getLong(i * 8); + assertEquals(i, val); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully read " + (i + 1) + " long words"); + } + } + System.out.println("Successfully read " + (bufSize / 8) + " long words"); + } + System.out.println("Successfully released the large buffer."); + } + + public static void main(String[] args) { + testLargeArrowBuf(4 * 1024 * 1024 * 1024L); + testLargeArrowBuf(Integer.MAX_VALUE); + } +} diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java b/java/memory/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java new file mode 100644 index 00000000000..5a700679183 --- /dev/null +++ b/java/memory/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import io.netty.buffer.ArrowBuf; + +/** + * Test cases for {@link NettyAllocationManager}. + */ +public class TestNettyAllocationManager { + + static int CUSTOMIZED_ALLOCATION_CUTOFF_VALUE = 1024; + + private BaseAllocator createCustomizedAllocator() { + return new RootAllocator(BaseAllocator.configBuilder() + .allocationManagerFactory((accountingAllocator, requestedSize) -> + new NettyAllocationManager( + accountingAllocator, requestedSize, CUSTOMIZED_ALLOCATION_CUTOFF_VALUE)).build()); + } + + private void readWriteArrowBuf(ArrowBuf buffer) { + // write buffer + for (long i = 0; i < buffer.capacity() / 8; i++) { + buffer.setLong(i * 8, i); + } + + // read buffer + for (long i = 0; i < buffer.capacity() / 8; i++) { + long val = buffer.getLong(i * 8); + assertEquals(i, val); + } + } + + /** + * Test the allocation strategy for small buffers.. + */ + @Test + public void testSmallBufferAllocation() { + final long bufSize = CUSTOMIZED_ALLOCATION_CUTOFF_VALUE - 512L; + try (BaseAllocator allocator = createCustomizedAllocator(); + ArrowBuf buffer = allocator.buffer(bufSize)) { + + assertTrue(buffer.getReferenceManager() instanceof BufferLedger); + BufferLedger bufferLedger = (BufferLedger) buffer.getReferenceManager(); + + // make sure we are using netty allocation manager + AllocationManager allocMgr = bufferLedger.getAllocationManager(); + assertTrue(allocMgr instanceof NettyAllocationManager); + NettyAllocationManager nettyMgr = (NettyAllocationManager) allocMgr; + + // for the small buffer allocation strategy, the chunk is not null + assertNotNull(nettyMgr.getMemoryChunk()); + + readWriteArrowBuf(buffer); + } + } + + /** + * Test the allocation strategy for large buffers.. + */ + @Test + public void testLargeBufferAllocation() { + final long bufSize = CUSTOMIZED_ALLOCATION_CUTOFF_VALUE + 1024L; + try (BaseAllocator allocator = createCustomizedAllocator(); + ArrowBuf buffer = allocator.buffer(bufSize)) { + assertTrue(buffer.getReferenceManager() instanceof BufferLedger); + BufferLedger bufferLedger = (BufferLedger) buffer.getReferenceManager(); + + // make sure we are using netty allocation manager + AllocationManager allocMgr = bufferLedger.getAllocationManager(); + assertTrue(allocMgr instanceof NettyAllocationManager); + NettyAllocationManager nettyMgr = (NettyAllocationManager) allocMgr; + + // for the large buffer allocation strategy, the chunk is null + assertNull(nettyMgr.getMemoryChunk()); + + readWriteArrowBuf(buffer); + } + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index ede382110de..d8684714e78 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -124,7 +124,7 @@ protected static int getValidityBufferSizeFromCount(final int valueCount) { } /* round up bytes for the validity buffer for the given valueCount */ - private static long roundUp8ForValidityBuffer(int valueCount) { + private static long roundUp8ForValidityBuffer(long valueCount) { return ((valueCount + 63) >> 6) << 3; } @@ -140,7 +140,7 @@ long computeCombinedBufferSize(int valueCount, int typeWidth) { // for boolean type, value-buffer and validity-buffer are of same size. bufferSize *= 2; } else { - bufferSize += DataSizeRoundingUtil.roundUpTo8Multiple(valueCount * typeWidth); + bufferSize += DataSizeRoundingUtil.roundUpTo8Multiple((long) valueCount * typeWidth); } return BaseAllocator.nextPowerOfTwo(bufferSize); } @@ -170,16 +170,16 @@ DataAndValidityBuffers allocFixedDataAndValidityBufs(int valueCount, int typeWid long bufferSize = computeCombinedBufferSize(valueCount, typeWidth); assert bufferSize <= MAX_ALLOCATION_SIZE; - int validityBufferSize; - int dataBufferSize; + long validityBufferSize; + long dataBufferSize; if (typeWidth == 0) { - validityBufferSize = dataBufferSize = (int) (bufferSize / 2); + validityBufferSize = dataBufferSize = bufferSize / 2; } else { // Due to roundup to power-of-2 allocation, the bufferSize could be greater than the // requested size. Utilize the allocated buffer fully.; - int actualCount = (int) ((bufferSize * 8.0) / (8 * typeWidth + 1)); + long actualCount = (long) ((bufferSize * 8.0) / (8 * typeWidth + 1)); do { - validityBufferSize = (int) roundUp8ForValidityBuffer(actualCount); + validityBufferSize = roundUp8ForValidityBuffer(actualCount); dataBufferSize = DataSizeRoundingUtil.roundUpTo8Multiple(actualCount * typeWidth); if (validityBufferSize + dataBufferSize <= bufferSize) { break; @@ -191,14 +191,14 @@ DataAndValidityBuffers allocFixedDataAndValidityBufs(int valueCount, int typeWid /* allocate combined buffer */ - ArrowBuf combinedBuffer = allocator.buffer((int) bufferSize); + ArrowBuf combinedBuffer = allocator.buffer(bufferSize); /* slice into requested lengths */ ArrowBuf dataBuf = null; ArrowBuf validityBuf = null; - int bufferOffset = 0; + long bufferOffset = 0; for (int numBuffers = 0; numBuffers < 2; ++numBuffers) { - int len = (numBuffers == 0 ? dataBufferSize : validityBufferSize); + long len = (numBuffers == 0 ? dataBufferSize : validityBufferSize); ArrowBuf buf = combinedBuffer.slice(bufferOffset, len); buf.getReferenceManager().retain(); buf.readerIndex(0); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java index ac2015ea4ef..b392eb4da88 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java @@ -112,7 +112,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -128,7 +128,7 @@ public void get(int index, NullableBigIntHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -141,7 +141,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } @@ -153,7 +153,7 @@ public Long getObject(int index) { private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** @@ -275,7 +275,7 @@ public void setSafe(int index, int isSet, long value) { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java index 9448a1a1453..dab08e03dfa 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java @@ -114,7 +114,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableDateDayHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } } @@ -155,7 +155,7 @@ public Integer getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** @@ -279,7 +279,7 @@ public void setSafe(int index, int isSet, int value) { * @return value stored at the index. */ public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java index d893ba4f331..c3037ef1a2f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java @@ -116,7 +116,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -132,7 +132,7 @@ public void get(int index, NullableDateMilliHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -145,7 +145,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long millis = valueBuffer.getLong(index * TYPE_WIDTH); + final long millis = valueBuffer.getLong((long) index * TYPE_WIDTH); return DateUtility.getLocalDateTimeFromEpochMilli(millis); } } @@ -158,7 +158,7 @@ public LocalDateTime getObject(int index) { private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** @@ -282,7 +282,7 @@ public void setSafe(int index, int isSet, long value) { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index 0dc1bdd3fb4..0cd9d139de5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -126,7 +126,7 @@ public ArrowBuf get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + return valueBuffer.slice((long) index * TYPE_WIDTH, TYPE_WIDTH); } /** @@ -192,7 +192,7 @@ public int getScale() { */ public void set(int index, ArrowBuf buffer) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); } /** @@ -215,9 +215,9 @@ public void setBigEndian(int index, byte[] value) { final int length = value.length; // do the bound check. - valueBuffer.checkBytes(index * TYPE_WIDTH, (index + 1) * TYPE_WIDTH); + valueBuffer.checkBytes((long) index * TYPE_WIDTH, (long) (index + 1) * TYPE_WIDTH); - long outAddress = valueBuffer.memoryAddress() + index * TYPE_WIDTH; + long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH; // swap bytes to convert BE to LE for (int byteIdx = 0; byteIdx < length; ++byteIdx) { PlatformDependent.putByte(outAddress + byteIdx, value[length - 1 - byteIdx]); @@ -248,7 +248,7 @@ public void setBigEndian(int index, byte[] value) { */ public void set(int index, int start, ArrowBuf buffer) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, buffer, start, TYPE_WIDTH); } /** @@ -264,10 +264,10 @@ public void setSafe(int index, int start, ArrowBuf buffer, int length) { // do the bound checks. buffer.checkBytes(start, start + length); - valueBuffer.checkBytes(index * TYPE_WIDTH, (index + 1) * TYPE_WIDTH); + valueBuffer.checkBytes((long) index * TYPE_WIDTH, (long) (index + 1) * TYPE_WIDTH); long inAddress = buffer.memoryAddress() + start; - long outAddress = valueBuffer.memoryAddress() + index * TYPE_WIDTH; + long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH; PlatformDependent.copyMemory(inAddress, outAddress, length); // sign extend if (length < 16) { @@ -291,11 +291,11 @@ public void setBigEndianSafe(int index, int start, ArrowBuf buffer, int length) // do the bound checks. buffer.checkBytes(start, start + length); - valueBuffer.checkBytes(index * TYPE_WIDTH, (index + 1) * TYPE_WIDTH); + valueBuffer.checkBytes((long) index * TYPE_WIDTH, (long) (index + 1) * TYPE_WIDTH); // not using buffer.getByte() to avoid boundary checks for every byte. long inAddress = buffer.memoryAddress() + start; - long outAddress = valueBuffer.memoryAddress() + index * TYPE_WIDTH; + long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH; // swap bytes to convert BE to LE for (int byteIdx = 0; byteIdx < length; ++byteIdx) { byte val = PlatformDependent.getByte((inAddress + length - 1) - byteIdx); @@ -345,7 +345,7 @@ public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentE throw new IllegalArgumentException(); } else if (holder.isSet > 0) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); } else { BitVectorHelper.unsetBit(validityBuffer, index); } @@ -359,7 +359,7 @@ public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentE */ public void set(int index, DecimalHolder holder) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java index 533d331de95..9c09347011a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java @@ -110,7 +110,7 @@ public MinorType getMinorType() { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } /** @@ -123,7 +123,7 @@ public ArrowBuf get(int index) throws IllegalStateException { if (isSet(index) == 0) { return null; } - return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + return valueBuffer.slice((long) index * TYPE_WIDTH, TYPE_WIDTH); } /** @@ -207,7 +207,7 @@ private StringBuilder getAsStringBuilderHelper(int index) { */ public void set(int index, ArrowBuf value) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, value, 0, TYPE_WIDTH); } /** @@ -217,7 +217,7 @@ public void set(int index, ArrowBuf value) { * @param value The duration value (in the timeunit associated with this vector) */ public void set(int index, long value) { - final int offsetIndex = index * TYPE_WIDTH; + final long offsetIndex = (long) index * TYPE_WIDTH; BitVectorHelper.setBit(validityBuffer, index); valueBuffer.setLong(offsetIndex, value); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java index aad185b58c5..630522b72bf 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java @@ -117,7 +117,7 @@ public byte[] get(int index) { return null; } final byte[] dst = new byte[byteWidth]; - valueBuffer.getBytes(index * byteWidth, dst, 0, byteWidth); + valueBuffer.getBytes((long) index * byteWidth, dst, 0, byteWidth); return dst; } @@ -136,7 +136,7 @@ public void get(int index, NullableFixedSizeBinaryHolder holder) { return; } holder.isSet = 1; - holder.buffer = valueBuffer.slice(index * byteWidth, byteWidth); + holder.buffer = valueBuffer.slice((long) index * byteWidth, byteWidth); } /** @@ -167,7 +167,7 @@ public void set(int index, byte[] value) { Preconditions.checkNotNull(value, "expecting a valid byte array"); assert byteWidth <= value.length; BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * byteWidth, value, 0, byteWidth); + valueBuffer.setBytes((long) index * byteWidth, value, 0, byteWidth); } /** @@ -205,7 +205,7 @@ public void set(int index, ArrowBuf buffer) { assert index >= 0; assert byteWidth <= buffer.capacity(); BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * byteWidth, buffer, 0, byteWidth); + valueBuffer.setBytes((long) index * byteWidth, buffer, 0, byteWidth); } /** @@ -316,7 +316,7 @@ public void setSafe(int index, NullableFixedSizeBinaryHolder holder) { */ public static byte[] get(final ArrowBuf buffer, final int index, final int byteWidth) { final byte[] dst = new byte[byteWidth]; - buffer.getBytes(index * byteWidth, dst, 0, byteWidth); + buffer.getBytes((long) index * byteWidth, dst, 0, byteWidth); return dst; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java index 74fbc32ca86..b083f3085fe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java @@ -114,7 +114,7 @@ public float get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getFloat(index * TYPE_WIDTH); + return valueBuffer.getFloat((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableFloat4Holder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getFloat(index * TYPE_WIDTH); + holder.value = valueBuffer.getFloat((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Float getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getFloat(index * TYPE_WIDTH); + return valueBuffer.getFloat((long) index * TYPE_WIDTH); } } @@ -155,7 +155,7 @@ public Float getObject(int index) { private void setValue(int index, float value) { - valueBuffer.setFloat(index * TYPE_WIDTH, value); + valueBuffer.setFloat((long) index * TYPE_WIDTH, value); } /** @@ -279,7 +279,7 @@ public void setSafe(int index, int isSet, float value) { * @return value stored at the index. */ public static float get(final ArrowBuf buffer, final int index) { - return buffer.getFloat(index * TYPE_WIDTH); + return buffer.getFloat((long) index * TYPE_WIDTH); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java index 9ca40616982..80da81f5fbe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java @@ -114,7 +114,7 @@ public double get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getDouble(index * TYPE_WIDTH); + return valueBuffer.getDouble((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableFloat8Holder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getDouble(index * TYPE_WIDTH); + holder.value = valueBuffer.getDouble((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Double getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getDouble(index * TYPE_WIDTH); + return valueBuffer.getDouble((long) index * TYPE_WIDTH); } } @@ -156,7 +156,7 @@ public Double getObject(int index) { private void setValue(int index, double value) { - valueBuffer.setDouble(index * TYPE_WIDTH, value); + valueBuffer.setDouble((long) index * TYPE_WIDTH, value); } /** @@ -280,7 +280,7 @@ public void setSafe(int index, int isSet, double value) { * @return value stored at the index. */ public static double get(final ArrowBuf buffer, final int index) { - return buffer.getDouble(index * TYPE_WIDTH); + return buffer.getDouble((long) index * TYPE_WIDTH); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java index e91e0eaa0e2..c2b9ee4b2d8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java @@ -114,7 +114,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableIntHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } } @@ -155,7 +155,7 @@ public Integer getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** @@ -279,7 +279,7 @@ public void setSafe(int index, int isSet, int value) { * @return value stored at the index. */ public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java index b67d697b989..c5f85a1c0f6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java @@ -118,7 +118,7 @@ public MinorType getMinorType() { * @return day value stored at the index. */ public static int getDays(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } /** @@ -132,7 +132,7 @@ public static int getDays(final ArrowBuf buffer, final int index) { * @return milliseconds value stored at the index. */ public static int getMilliseconds(final ArrowBuf buffer, final int index) { - return buffer.getInt((index * TYPE_WIDTH) + MILLISECOND_OFFSET); + return buffer.getInt((long) index * TYPE_WIDTH + MILLISECOND_OFFSET); } /** @@ -145,7 +145,7 @@ public ArrowBuf get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { return null; } - return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + return valueBuffer.slice((long) index * TYPE_WIDTH, TYPE_WIDTH); } /** @@ -160,7 +160,7 @@ public void get(int index, NullableIntervalDayHolder holder) { holder.isSet = 0; return; } - final int startIndex = index * TYPE_WIDTH; + final long startIndex = (long) index * TYPE_WIDTH; holder.isSet = 1; holder.days = valueBuffer.getInt(startIndex); holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); @@ -176,7 +176,7 @@ public Duration getObject(int index) { if (isSet(index) == 0) { return null; } else { - final int startIndex = index * TYPE_WIDTH; + final long startIndex = (long) index * TYPE_WIDTH; final int days = valueBuffer.getInt(startIndex); final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); return Duration.ofDays(days).plusMillis(milliseconds); @@ -199,7 +199,7 @@ public StringBuilder getAsStringBuilder(int index) { } private StringBuilder getAsStringBuilderHelper(int index) { - final int startIndex = index * TYPE_WIDTH; + final long startIndex = (long) index * TYPE_WIDTH; final int days = valueBuffer.getInt(startIndex); int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); @@ -238,7 +238,7 @@ private StringBuilder getAsStringBuilderHelper(int index) { */ public void set(int index, ArrowBuf value) { BitVectorHelper.setBit(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); + valueBuffer.setBytes((long) index * TYPE_WIDTH, value, 0, TYPE_WIDTH); } /** @@ -249,7 +249,7 @@ public void set(int index, ArrowBuf value) { * @param milliseconds milliseconds for the interval */ public void set(int index, int days, int milliseconds) { - final int offsetIndex = index * TYPE_WIDTH; + final long offsetIndex = (long) index * TYPE_WIDTH; BitVectorHelper.setBit(validityBuffer, index); valueBuffer.setInt(offsetIndex, days); valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java index 25c7535c005..4bb448fc037 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java @@ -117,7 +117,7 @@ public MinorType getMinorType() { * @return value stored at the index. */ public static int getTotalMonths(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -146,7 +146,7 @@ public void get(int index, NullableIntervalYearHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -159,7 +159,7 @@ public Period getObject(int index) { if (isSet(index) == 0) { return null; } else { - final int interval = valueBuffer.getInt(index * TYPE_WIDTH); + final int interval = valueBuffer.getInt((long) index * TYPE_WIDTH); // TODO: verify interval is in months return Period.ofMonths(interval); } @@ -181,7 +181,7 @@ public StringBuilder getAsStringBuilder(int index) { } private StringBuilder getAsStringBuilderHelper(int index) { - int value = valueBuffer.getInt(index * TYPE_WIDTH); + int value = valueBuffer.getInt((long) index * TYPE_WIDTH); final int years = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths); final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths); @@ -204,7 +204,7 @@ private StringBuilder getAsStringBuilderHelper(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java index 724daf59f15..b0453fc58db 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java @@ -114,7 +114,7 @@ public short get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getShort(index * TYPE_WIDTH); + return valueBuffer.getShort((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableSmallIntHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getShort(index * TYPE_WIDTH); + holder.value = valueBuffer.getShort((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Short getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getShort(index * TYPE_WIDTH); + return valueBuffer.getShort((long) index * TYPE_WIDTH); } } @@ -155,11 +155,11 @@ public Short getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setShort(index * TYPE_WIDTH, value); + valueBuffer.setShort((long) index * TYPE_WIDTH, value); } private void setValue(int index, short value) { - valueBuffer.setShort(index * TYPE_WIDTH, value); + valueBuffer.setShort((long) index * TYPE_WIDTH, value); } /** @@ -307,7 +307,7 @@ public void setSafe(int index, int isSet, short value) { * @return value stored at the index. */ public static short get(final ArrowBuf buffer, final int index) { - return buffer.getShort(index * TYPE_WIDTH); + return buffer.getShort((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java index b1ab36a6824..86adc2ad43d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java @@ -114,7 +114,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableTimeMicroHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } @@ -155,7 +155,7 @@ public Long getObject(int index) { private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** @@ -279,7 +279,7 @@ public void setSafe(int index, int isSet, long value) { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java index 8c7943dfd0d..93ebf1952ee 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java @@ -116,7 +116,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -132,7 +132,7 @@ public void get(int index, NullableTimeMilliHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -145,7 +145,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } - final int millis = valueBuffer.getInt(index * TYPE_WIDTH); + final int millis = valueBuffer.getInt((long) index * TYPE_WIDTH); // TODO: this doesn't seem right, time not from epoch return DateUtility.getLocalDateTimeFromEpochMilli(millis); } @@ -159,7 +159,7 @@ public LocalDateTime getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** @@ -284,7 +284,7 @@ public void setSafe(int index, int isSet, int value) { * @return value stored at the index. */ public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java index b711131e177..b296ba35772 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java @@ -114,7 +114,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableTimeNanoHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } @@ -156,7 +156,7 @@ public Long getObject(int index) { private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** @@ -280,7 +280,7 @@ public void setSafe(int index, int isSet, long value) { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java index 69e96b94084..eca4f304ef0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java @@ -114,7 +114,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -130,7 +130,7 @@ public void get(int index, NullableTimeSecHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -143,7 +143,7 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } } @@ -156,7 +156,7 @@ public Integer getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** @@ -280,7 +280,7 @@ public void setSafe(int index, int isSet, int value) { * @return value stored at the index. */ public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); + return buffer.getInt((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java index 9c408e33610..ce34da99fe0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java @@ -120,7 +120,7 @@ public void get(int index, NullableTimeStampMicroTZHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -133,7 +133,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java index 0b822259f55..d9cb2b9c13a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java @@ -116,7 +116,7 @@ public void get(int index, NullableTimeStampMicroHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -129,7 +129,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long micros = valueBuffer.getLong(index * TYPE_WIDTH); + final long micros = valueBuffer.getLong((long) index * TYPE_WIDTH); return DateUtility.getLocalDateTimeFromEpochMicro(micros); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java index 47c59ac373e..b2870ba4f27 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java @@ -120,7 +120,7 @@ public void get(int index, NullableTimeStampMilliTZHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -133,7 +133,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java index 23c542e231e..55928154d5f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java @@ -116,7 +116,7 @@ public void get(int index, NullableTimeStampMilliHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -129,7 +129,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long millis = valueBuffer.getLong(index * TYPE_WIDTH); + final long millis = valueBuffer.getLong((long) index * TYPE_WIDTH); return DateUtility.getLocalDateTimeFromEpochMilli(millis); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java index b91f352e960..ff0987901a9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java @@ -120,7 +120,7 @@ public void get(int index, NullableTimeStampNanoTZHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -133,7 +133,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java index 4873b34bc94..5c2dd663997 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java @@ -116,7 +116,7 @@ public void get(int index, NullableTimeStampNanoHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -129,7 +129,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long nanos = valueBuffer.getLong(index * TYPE_WIDTH); + final long nanos = valueBuffer.getLong((long) index * TYPE_WIDTH); return DateUtility.getLocalDateTimeFromEpochNano(nanos); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java index 78bda17cfba..8034e9bf74d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java @@ -120,7 +120,7 @@ public void get(int index, NullableTimeStampSecTZHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -133,7 +133,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java index 7dfedf8c883..59660a25f03 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java @@ -116,7 +116,7 @@ public void get(int index, NullableTimeStampSecHolder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -129,7 +129,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long secs = valueBuffer.getLong(index * TYPE_WIDTH); + final long secs = valueBuffer.getLong((long) index * TYPE_WIDTH); final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs); return DateUtility.getLocalDateTimeFromEpochMilli(millis); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java index 271fdb5ee0c..b2202994306 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java @@ -74,7 +74,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } @@ -86,7 +86,7 @@ public long get(int index) throws IllegalStateException { protected void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** @@ -154,7 +154,7 @@ public void setSafe(int index, int isSet, long value) { * @return value stored at the index. */ public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); + return buffer.getLong((long) index * TYPE_WIDTH); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java index 349a50a69a6..f2b834d9382 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java @@ -80,7 +80,7 @@ public MinorType getMinorType() { * @return value stored at the index. */ public static char get(final ArrowBuf buffer, final int index) { - return buffer.getChar(index * TYPE_WIDTH); + return buffer.getChar((long) index * TYPE_WIDTH); } /** @@ -93,7 +93,7 @@ public char get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getChar(index * TYPE_WIDTH); + return valueBuffer.getChar((long) index * TYPE_WIDTH); } /** @@ -109,7 +109,7 @@ public void get(int index, NullableUInt2Holder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getChar(index * TYPE_WIDTH); + holder.value = valueBuffer.getChar((long) index * TYPE_WIDTH); } /** @@ -122,7 +122,7 @@ public Character getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getChar(index * TYPE_WIDTH); + return valueBuffer.getChar((long) index * TYPE_WIDTH); } } @@ -135,11 +135,11 @@ public Character getObject(int index) { private void setValue(int index, int value) { - valueBuffer.setChar(index * TYPE_WIDTH, value); + valueBuffer.setChar((long) index * TYPE_WIDTH, value); } private void setValue(int index, char value) { - valueBuffer.setChar(index * TYPE_WIDTH, value); + valueBuffer.setChar((long) index * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java index a747026a176..0c99b7dcf08 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java @@ -83,7 +83,7 @@ public MinorType getMinorType() { * @return value stored at the index. */ public static long getNoOverflow(final ArrowBuf buffer, final int index) { - long l = buffer.getInt(index * TYPE_WIDTH); + long l = buffer.getInt((long) index * TYPE_WIDTH); return (0x00000000FFFFFFFFL) & l; } @@ -97,7 +97,7 @@ public int get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -113,7 +113,7 @@ public void get(int index, NullableUInt4Holder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + holder.value = valueBuffer.getInt((long) index * TYPE_WIDTH); } /** @@ -126,7 +126,7 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getInt(index * TYPE_WIDTH); + return valueBuffer.getInt((long) index * TYPE_WIDTH); } } @@ -153,7 +153,7 @@ public Long getObjectNoOverflow(int index) { private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); + valueBuffer.setInt((long) index * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java index 19d761b7857..2660e873a9f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java @@ -87,7 +87,7 @@ public MinorType getMinorType() { * @return value stored at the index. */ public static BigInteger getNoOverflow(final ArrowBuf buffer, final int index) { - BigInteger l = BigInteger.valueOf(buffer.getLong(index * TYPE_WIDTH)); + BigInteger l = BigInteger.valueOf(buffer.getLong((long) index * TYPE_WIDTH)); return SAFE_CONVERSION_MASK.and(l); } @@ -102,7 +102,7 @@ public long get(int index) throws IllegalStateException { if (NULL_CHECKING_ENABLED && isSet(index) == 0) { throw new IllegalStateException("Value at index is null"); } - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -118,7 +118,7 @@ public void get(int index, NullableUInt8Holder holder) { return; } holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + holder.value = valueBuffer.getLong((long) index * TYPE_WIDTH); } /** @@ -131,7 +131,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return valueBuffer.getLong(index * TYPE_WIDTH); + return valueBuffer.getLong((long) index * TYPE_WIDTH); } } @@ -158,7 +158,7 @@ public BigInteger getObjectNoOverflow(int index) { private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); + valueBuffer.setLong((long) index * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java index 2703d261219..83194cbc22e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java @@ -73,7 +73,7 @@ public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int sca */ public static byte[] getByteArrayFromArrowBuf(ArrowBuf bytebuf, int index) { final byte[] value = new byte[DECIMAL_BYTE_LENGTH]; - final int startIndex = index * DECIMAL_BYTE_LENGTH; + final long startIndex = (long) index * DECIMAL_BYTE_LENGTH; bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH); return value; } @@ -127,7 +127,7 @@ public static void writeBigDecimalToArrowBuf(BigDecimal value, ArrowBuf bytebuf, * Write the given long to the ArrowBuf at the given value index. */ public static void writeLongToArrowBuf(long value, ArrowBuf bytebuf, int index) { - final long addressOfValue = bytebuf.memoryAddress() + index * DECIMAL_BYTE_LENGTH; + final long addressOfValue = bytebuf.memoryAddress() + (long) index * DECIMAL_BYTE_LENGTH; PlatformDependent.putLong(addressOfValue, value); final long padValue = Long.signum(value) == -1 ? -1L : 0L; PlatformDependent.putLong(addressOfValue + Long.BYTES, padValue); @@ -143,7 +143,7 @@ public static void writeByteArrayToArrowBuf(byte[] bytes, ArrowBuf bytebuf, int } private static void writeByteArrayToArrowBufHelper(byte[] bytes, ArrowBuf bytebuf, int index) { - final int startIndex = index * DECIMAL_BYTE_LENGTH; + final long startIndex = (long) index * DECIMAL_BYTE_LENGTH; if (bytes.length > DECIMAL_BYTE_LENGTH) { throw new UnsupportedOperationException("Decimal size greater than 16 bytes"); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java new file mode 100644 index 00000000000..e128d40b0a6 --- /dev/null +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; + +import io.netty.buffer.ArrowBuf; + +/** + * Integration test for a vector with a large (more than 2GB) {@link io.netty.buffer.ArrowBuf} as + * the data buffer. + * To run this test, please make sure there is at least 4GB free memory in the system. + *

+ * Please note that this is not a standard test case, so please run it by manually invoking the + * main method. + *

+ */ +public class TestLargeVector { + + private static void testLargeLongVector() { + System.out.println("Testing large big int vector."); + + final long bufSize = 4 * 1024 * 1024 * 1024L; + final int vecLength = (int) (bufSize / BigIntVector.TYPE_WIDTH); + + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + BigIntVector largeVec = new BigIntVector("vec", allocator)) { + largeVec.allocateNew(vecLength); + + System.out.println("Successfully allocated a vector with capacity " + vecLength); + + for (int i = 0; i < vecLength; i++) { + largeVec.set(i, i * 10L); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully written " + (i + 1) + " values"); + } + } + System.out.println("Successfully written " + vecLength + " values"); + + for (int i = 0; i < vecLength; i++) { + long val = largeVec.get(i); + assertEquals(i * 10L, val); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully read " + (i + 1) + " values"); + } + } + System.out.println("Successfully read " + vecLength + " values"); + } + System.out.println("Successfully released the large vector."); + } + + private static void testLargeIntVector() { + System.out.println("Testing large int vector."); + + final long bufSize = 4 * 1024 * 1024 * 1024L; + final int vecLength = (int) (bufSize / IntVector.TYPE_WIDTH); + + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + IntVector largeVec = new IntVector("vec", allocator)) { + largeVec.allocateNew(vecLength); + + System.out.println("Successfully allocated a vector with capacity " + vecLength); + + for (int i = 0; i < vecLength; i++) { + largeVec.set(i, i); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully written " + (i + 1) + " values"); + } + } + System.out.println("Successfully written " + vecLength + " values"); + + for (int i = 0; i < vecLength; i++) { + long val = largeVec.get(i); + assertEquals(i, val); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully read " + (i + 1) + " values"); + } + } + System.out.println("Successfully read " + vecLength + " values"); + } + System.out.println("Successfully released the large vector."); + } + + private static void testLargeDecimalVector() { + System.out.println("Testing large decimal vector."); + + final long bufSize = 4 * 1024 * 1024 * 1024L; + final int vecLength = (int) (bufSize / DecimalVector.TYPE_WIDTH); + + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + DecimalVector largeVec = new DecimalVector("vec", allocator, 38, 16)) { + largeVec.allocateNew(vecLength); + + System.out.println("Successfully allocated a vector with capacity " + vecLength); + + for (int i = 0; i < vecLength; i++) { + largeVec.set(i, 0); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully written " + (i + 1) + " values"); + } + } + System.out.println("Successfully written " + vecLength + " values"); + + for (int i = 0; i < vecLength; i++) { + ArrowBuf buf = largeVec.get(i); + assertEquals(buf.capacity(), DecimalVector.TYPE_WIDTH); + assertEquals(0, buf.getLong(0)); + assertEquals(0, buf.getLong(8)); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully read " + (i + 1) + " values"); + } + } + System.out.println("Successfully read " + vecLength + " values"); + } + System.out.println("Successfully released the large vector."); + } + + private static void testLargeFixedSizeBinaryVector() { + System.out.println("Testing large fixed size binary vector."); + + final long bufSize = 4 * 1024 * 1024 * 1024L; + final int typeWidth = 8; + final int vecLength = (int) (bufSize / typeWidth); + + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + FixedSizeBinaryVector largeVec = new FixedSizeBinaryVector("vec", allocator, typeWidth)) { + largeVec.allocateNew(vecLength); + + System.out.println("Successfully allocated a vector with capacity " + vecLength); + + byte[] value = new byte[] {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}; + for (int i = 0; i < vecLength; i++) { + largeVec.set(i, value); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully written " + (i + 1) + " values"); + } + } + System.out.println("Successfully written " + vecLength + " values"); + + for (int i = 0; i < vecLength; i++) { + byte[] buf = largeVec.get(i); + assertEquals(typeWidth, buf.length); + assertArrayEquals(buf, value); + + if ((i + 1) % 10000 == 0) { + System.out.println("Successfully read " + (i + 1) + " values"); + } + } + System.out.println("Successfully read " + vecLength + " values"); + } + System.out.println("Successfully released the large vector."); + } + + public static void main(String[] args) { + testLargeLongVector(); + testLargeIntVector(); + testLargeDecimalVector(); + testLargeFixedSizeBinaryVector(); + } +}