diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml index d9c618250c4..8c3e65a8e0a 100644 --- a/java/adapter/avro/pom.xml +++ b/java/adapter/avro/pom.xml @@ -30,10 +30,9 @@ org.apache.arrow - arrow-memory + arrow-memory-netty ${project.version} - org.apache.arrow diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index eec274f773f..31f4ed7d208 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -30,10 +30,9 @@ org.apache.arrow - arrow-memory + arrow-memory-netty ${project.version} - org.apache.arrow diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index b753cf6f897..cfe8a62d306 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -15,7 +15,7 @@ org.apache.arrow - arrow-memory + arrow-memory-netty ${project.version} compile diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml index 774fdd52815..4bc0580e71d 100644 --- a/java/algorithm/pom.xml +++ b/java/algorithm/pom.xml @@ -35,7 +35,7 @@ org.apache.arrow - arrow-memory + arrow-memory-netty ${project.version} diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index d53bd8e3e91..7c8f9d7d6e7 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -38,7 +38,7 @@ org.apache.arrow - arrow-memory + arrow-memory-netty ${project.version} diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml index 62256777809..62db6ffb8d9 100644 --- a/java/flight/flight-grpc/pom.xml +++ b/java/flight/flight-grpc/pom.xml @@ -61,7 +61,7 @@ org.apache.arrow - arrow-memory + arrow-memory-netty ${project.version} compile diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index 5d3fcd0d09c..8aa881f0fde 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -33,7 +33,7 @@ org.apache.arrow - arrow-memory + arrow-memory-netty ${project.version} diff --git a/java/memory/pom.xml b/java/memory/pom.xml index 3cceb9e7cef..528243bc305 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -25,14 +25,6 @@ com.google.code.findbugs jsr305 - - io.netty - netty-buffer - - - io.netty - netty-common - org.slf4j slf4j-api diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java index 84a8f63e591..8e9ef428b41 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java @@ -55,7 +55,7 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato private final Object DEBUG_LOCK = DEBUG ? new Object() : null; final AllocationListener listener; private final BaseAllocator parentAllocator; - private final ArrowByteBufAllocator thisAsByteBufAllocator; + //private final ArrowByteBufAllocator thisAsByteBufAllocator; private final Map childAllocators; private final ArrowBuf empty; // members used purely for debugging @@ -99,7 +99,7 @@ protected BaseAllocator( this.parentAllocator = parentAllocator; this.name = name; - this.thisAsByteBufAllocator = new ArrowByteBufAllocator(this); + //this.thisAsByteBufAllocator = new ArrowByteBufAllocator(this); this.childAllocators = Collections.synchronizedMap(new IdentityHashMap<>()); if (DEBUG) { @@ -281,7 +281,7 @@ public ArrowBuf buffer(final long initialRequestSize) { } private ArrowBuf createEmpty() { - return new ArrowBuf(ReferenceManager.NO_OP, null, 0, NettyAllocationManager.EMPTY.memoryAddress(), true); + return new ArrowBuf(ReferenceManager.NO_OP, null, 0, 0, true); } @Override @@ -355,10 +355,10 @@ private AllocationManager newAllocationManager(BaseAllocator accountingAllocator return allocationManagerFactory.create(accountingAllocator, size); } - @Override - public ArrowByteBufAllocator getAsByteBufAllocator() { - return thisAsByteBufAllocator; - } + //@Override + //public ArrowByteBufAllocator getAsByteBufAllocator() { + // return thisAsByteBufAllocator; + //} @Override public BufferAllocator newChildAllocator( @@ -765,7 +765,7 @@ abstract static class Config { */ @Value.Default AllocationManager.Factory getAllocationManagerFactory() { - return NettyAllocationManager.FACTORY; + return TrivialAllocationManager.FACTORY; } /** diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java index 5eb56d6b05d..9146011a242 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java @@ -43,21 +43,45 @@ public RootAllocator(final AllocationListener listener, final long limit) { /** * Constructor. * - * @param listener the allocation listener - * @param limit max allocation size in bytes - * @param roundingPolicy the policy for rounding the buffer size + * @param listener the allocation listener + * @param limit max allocation size in bytes + * @param roundingPolicy the policy for rounding the buffer size */ public RootAllocator(final AllocationListener listener, final long limit, RoundingPolicy roundingPolicy) { this(configBuilder() .listener(listener) .maxAllocation(limit) .roundingPolicy(roundingPolicy) + .allocationManagerFactory(getFactory()) .build() ); } public RootAllocator(Config config) { super(null, "ROOT", config); + + } + + private static AllocationManager.Factory getFactory() { + String className = System.getProperty("arrow.default.allocation-manager"); + if (className == null) { + try { + return getFactory("org.apache.arrow.memory.NettyAllocationManager"); + } catch (Throwable t) { + return getFactory("org.apache.arrow.memory.TrivialAllocationManager"); + } + } else { + return getFactory(className); + } + } + + private static AllocationManager.Factory getFactory(String className) { + try { + Class clazz = Class.forName(className); + return (AllocationManager.Factory) clazz.getConstructor().newInstance(); + } catch (Throwable t) { + throw new RuntimeException("Could not instantiate AllocationManager with type " + className, t); + } } /** diff --git a/java/memory/src/main/java/org/apache/arrow/memory/TrivialAllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/TrivialAllocationManager.java new file mode 100644 index 00000000000..d5b15163ae4 --- /dev/null +++ b/java/memory/src/main/java/org/apache/arrow/memory/TrivialAllocationManager.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import java.lang.reflect.Field; + +import org.apache.arrow.memory.util.LargeMemoryUtil; + +import sun.misc.Unsafe; + +/** + * trivial allocation manager for the case when no other allocation manager exists. + */ +public class TrivialAllocationManager extends AllocationManager { + + public static final Factory FACTORY = new Factory(); + + private final Unsafe unsafe = getUnsafe(); + private final long address; + private final int requestedSize; + + protected TrivialAllocationManager(BaseAllocator accountingAllocator, int requestedSize) { + super(accountingAllocator); + this.requestedSize = requestedSize; + address = unsafe.allocateMemory(requestedSize); + } + + @Override + protected long memoryAddress() { + return address; + } + + @Override + protected void release0() { + unsafe.setMemory(address, requestedSize, (byte) 0); + unsafe.freeMemory(address); + } + + @Override + public long getSize() { + return requestedSize; + } + + private Unsafe getUnsafe() { + Field f = null; + try { + f = Unsafe.class.getDeclaredField("theUnsafe"); + f.setAccessible(true); + return (Unsafe) f.get(null); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } finally { + if (f != null) { + f.setAccessible(false); + } + } + } + + /** + * Factory for creating {@link TrivialAllocationManager}. + */ + public static class Factory implements AllocationManager.Factory { + + @Override + public AllocationManager create(BaseAllocator accountingAllocator, long size) { + return new TrivialAllocationManager(accountingAllocator, LargeMemoryUtil.checkedCastToInt(size)); + } + } +} diff --git a/java/memory/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java b/java/memory/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java index 12505d91a50..a1971fff4f7 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java @@ -20,7 +20,6 @@ import java.lang.reflect.Field; import org.apache.arrow.memory.BaseAllocator; -import org.apache.arrow.memory.NettyAllocationManager; /** * The default rounding policy. That is, if the requested size is within the chunk size, @@ -38,7 +37,8 @@ public class DefaultRoundingPolicy implements RoundingPolicy { private DefaultRoundingPolicy() { try { - Field field = NettyAllocationManager.class.getDeclaredField("CHUNK_SIZE"); + Class clazz = Class.forName("org.apache.arrow.memory.NettyAllocationManager"); + Field field = clazz.getDeclaredField("CHUNK_SIZE"); field.setAccessible(true); chunkSize = (Long) field.get(null); } catch (Exception e) { diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java b/java/memory/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java index 4a83fd7426c..9a63f81fd0e 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java @@ -17,13 +17,12 @@ package org.apache.arrow.memory.util; +import java.lang.reflect.AccessibleObject; import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.security.AccessController; import java.security.PrivilegedAction; -import io.netty.buffer.ByteBuf; -import io.netty.util.internal.ReflectionUtil; import sun.misc.Unsafe; /** @@ -54,7 +53,7 @@ public class MemoryUtil { public Object run() { try { final Field unsafeField = Unsafe.class.getDeclaredField("theUnsafe"); - Throwable cause = ReflectionUtil.trySetAccessible(unsafeField, false); + Throwable cause = trySetAccessible(unsafeField, false); if (cause != null) { return cause; } @@ -84,7 +83,7 @@ public Object run() { } /** - * Given a {@link ByteBuf}, gets the address the underlying memory space. + * Given a {@link ByteBuffer}, gets the address the underlying memory space. * * @param buf the byte buffer. * @return address of the underlying memory. @@ -95,4 +94,45 @@ public static long getByteBufferAddress(ByteBuffer buf) { private MemoryUtil() { } + + /** + * Try to call {@link AccessibleObject#setAccessible(boolean)} but will catch any {@link SecurityException} and + * {@link java.lang.reflect.InaccessibleObjectException} and return it. + * The caller must check if it returns {@code null} and if not handle the returned exception. + */ + private static Throwable trySetAccessible(AccessibleObject object, boolean checkAccessible) { + if (checkAccessible && javaVersion() > 9) { + return new UnsupportedOperationException("Reflective setAccessible(true) disabled"); + } + try { + object.setAccessible(true); + return null; + } catch (SecurityException e) { + return e; + } catch (RuntimeException e) { + if ("java.lang.reflect.InaccessibleObjectException".equals(e.getClass().getName())) { + return e; + } + throw e; + } + } + + private static int javaVersion() { + return majorVersion(System.getProperty("java.specification.version", "1.6")); + } + + private static int majorVersion(final String javaSpecVersion) { + final String[] components = javaSpecVersion.split("\\."); + final int[] version = new int[components.length]; + for (int i = 0; i < components.length; i++) { + version[i] = Integer.parseInt(components[i]); + } + + if (version[0] == 1) { + assert version[1] >= 6; + return version[1]; + } else { + return version[0]; + } + } } diff --git a/java/netty-memory/pom.xml b/java/netty-memory/pom.xml new file mode 100644 index 00000000000..622a2f3adff --- /dev/null +++ b/java/netty-memory/pom.xml @@ -0,0 +1,45 @@ + + + + + arrow-java-root + org.apache.arrow + 1.0.0-SNAPSHOT + + 4.0.0 + arrow-memory-netty + Arrow Memory Netty + Netty implementation of core memory management libraries for Arrow ValueVectors. + + + + + io.netty + netty-buffer + + + io.netty + netty-common + + + org.slf4j + slf4j-api + + + org.apache.arrow + arrow-memory + ${project.version} + + + \ No newline at end of file diff --git a/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java b/java/netty-memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java similarity index 95% rename from java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java rename to java/netty-memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java index c2f3f68b0c0..6e63b193c16 100644 --- a/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java +++ b/java/netty-memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java @@ -41,7 +41,7 @@ public ByteBuf copy(int index, int length) { @Override public ByteBuf capacity(int newCapacity) { if (newCapacity > capacity()) { - ByteBuf newBuf = allocator.buffer(newCapacity).asNettyBuffer(); + ByteBuf newBuf = NettyArrowBuf.arrowBufToNettyBuf(allocator.buffer(newCapacity)); newBuf.writeBytes(buffer, 0, buffer.capacity()); newBuf.readerIndex(buffer.readerIndex()); newBuf.writerIndex(buffer.writerIndex()); diff --git a/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java b/java/netty-memory/src/main/java/io/netty/buffer/LargeBuffer.java similarity index 100% rename from java/memory/src/main/java/io/netty/buffer/LargeBuffer.java rename to java/netty-memory/src/main/java/io/netty/buffer/LargeBuffer.java diff --git a/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java b/java/netty-memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java similarity index 100% rename from java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java rename to java/netty-memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java diff --git a/java/memory/src/main/java/io/netty/buffer/NettyArrowBuf.java b/java/netty-memory/src/main/java/io/netty/buffer/NettyArrowBuf.java similarity index 93% rename from java/memory/src/main/java/io/netty/buffer/NettyArrowBuf.java rename to java/netty-memory/src/main/java/io/netty/buffer/NettyArrowBuf.java index c179c56b86e..e650ad6a46a 100644 --- a/java/memory/src/main/java/io/netty/buffer/NettyArrowBuf.java +++ b/java/netty-memory/src/main/java/io/netty/buffer/NettyArrowBuf.java @@ -163,12 +163,12 @@ public int capacity() { @Override public NettyArrowBuf slice() { - return arrowBuf.slice(readerIndex, writerIndex - readerIndex).asNettyBuffer(); + return NettyArrowBuf.arrowBufToNettyBuf(arrowBuf.slice(readerIndex, writerIndex - readerIndex)); } @Override public NettyArrowBuf slice(int index, int length) { - return arrowBuf.slice(index, length).asNettyBuffer(); + return NettyArrowBuf.arrowBufToNettyBuf(arrowBuf.slice(index, length)); } @Override @@ -600,4 +600,29 @@ public NettyArrowBuf setLong(int index, long value) { arrowBuf.setLong(index, value); return this; } + + /** + * Get a wrapper buffer to comply with Netty interfaces and + * can be used in RPC/RPC allocator code. + * @return netty compliant {@link NettyArrowBuf} + */ + public static NettyArrowBuf arrowBufToNettyBuf(ArrowBuf buf) { + + NettyArrowBuf nettyArrowBuf = new NettyArrowBuf( + buf, + buf.isEmpty() ? null : new ArrowByteBufAllocator(buf.getReferenceManager().getAllocator()), + checkedCastToInt(buf.capacity())); + nettyArrowBuf.readerIndex(checkedCastToInt(buf.readerIndex())); + nettyArrowBuf.writerIndex(checkedCastToInt(buf.writerIndex())); + return nettyArrowBuf; + } + + + public static ByteBuffer arrowBufToNioBuffer(ArrowBuf buf) { + return buf.isEmpty() ? ByteBuffer.allocateDirect(0) : arrowBufToNettyBuf(buf).nioBuffer(); + } + + public static ByteBuffer arrowBufToNioBuffer(ArrowBuf buf, long index, int length) { + return buf.isEmpty() ? ByteBuffer.allocateDirect(0) : arrowBufToNettyBuf(buf).nioBuffer(index, length); + } } diff --git a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/netty-memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java similarity index 99% rename from java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java rename to java/netty-memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java index d0a5a9945ce..ef4b38fd272 100644 --- a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java +++ b/java/netty-memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java @@ -51,7 +51,7 @@ public PooledByteBufAllocatorL() { } /** - * Returns a {@linkplain io.netty.buffer.UnsafeDirectLittleEndian} of the given size. + * Returns a {@linkplain UnsafeDirectLittleEndian} of the given size. */ public UnsafeDirectLittleEndian allocate(long size) { try { diff --git a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/java/netty-memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java similarity index 100% rename from java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java rename to java/netty-memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java b/java/netty-memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java similarity index 94% rename from java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java rename to java/netty-memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java index 96fe7eb1c51..489f03dd4dd 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java +++ b/java/netty-memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java @@ -21,6 +21,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.CompositeByteBuf; import io.netty.buffer.ExpandableByteBuf; +import io.netty.buffer.NettyArrowBuf; /** * An implementation of ByteBufAllocator that wraps a Arrow BufferAllocator. This allows the RPC @@ -56,7 +57,7 @@ public ByteBuf buffer() { @Override public ByteBuf buffer(int initialCapacity) { - return new ExpandableByteBuf(allocator.buffer(initialCapacity).asNettyBuffer(), allocator); + return new ExpandableByteBuf(NettyArrowBuf.arrowBufToNettyBuf(allocator.buffer(initialCapacity)), allocator); } @Override @@ -86,7 +87,7 @@ public ByteBuf directBuffer() { @Override public ByteBuf directBuffer(int initialCapacity) { - return allocator.buffer(initialCapacity).asNettyBuffer(); + return NettyArrowBuf.arrowBufToNettyBuf(allocator.buffer(initialCapacity)); } @Override diff --git a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java b/java/netty-memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java similarity index 99% rename from java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java rename to java/netty-memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java index 35b02f54ecd..14af3d4dfea 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java +++ b/java/netty-memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java @@ -75,7 +75,6 @@ public long getSize() { * Factory for creating {@link NettyAllocationManager}. */ public static class Factory implements AllocationManager.Factory { - private Factory() {} @Override public AllocationManager create(BaseAllocator accountingAllocator, long size) { diff --git a/java/memory/src/test/java/io/netty/buffer/TestNettyArrowBuf.java b/java/netty-memory/src/test/java/io/netty/buffer/TestNettyArrowBuf.java similarity index 88% rename from java/memory/src/test/java/io/netty/buffer/TestNettyArrowBuf.java rename to java/netty-memory/src/test/java/io/netty/buffer/TestNettyArrowBuf.java index afbfef54aad..5fd55b067c2 100644 --- a/java/memory/src/test/java/io/netty/buffer/TestNettyArrowBuf.java +++ b/java/netty-memory/src/test/java/io/netty/buffer/TestNettyArrowBuf.java @@ -32,7 +32,7 @@ public void testSliceWithoutArgs() { try (BufferAllocator allocator = new RootAllocator(128); ArrowBuf buf = allocator.buffer(20); ) { - NettyArrowBuf nettyBuf = buf.asNettyBuffer(); + NettyArrowBuf nettyBuf = NettyArrowBuf.arrowBufToNettyBuf(buf); nettyBuf.writerIndex(20); nettyBuf.readerIndex(10); NettyArrowBuf slicedBuffer = nettyBuf.slice(); @@ -46,7 +46,7 @@ public void testNioBuffer() { try (BufferAllocator allocator = new RootAllocator(128); ArrowBuf buf = allocator.buffer(20); ) { - NettyArrowBuf nettyBuf = buf.asNettyBuffer(); + NettyArrowBuf nettyBuf = NettyArrowBuf.arrowBufToNettyBuf(buf); ByteBuffer byteBuffer = nettyBuf.nioBuffer(4 ,6); // Nio Buffers should always be 0 indexed Assert.assertEquals(0, byteBuffer.position()); @@ -62,7 +62,7 @@ public void testInternalNioBuffer() { try (BufferAllocator allocator = new RootAllocator(128); ArrowBuf buf = allocator.buffer(20); ) { - NettyArrowBuf nettyBuf = buf.asNettyBuffer(); + NettyArrowBuf nettyBuf = NettyArrowBuf.arrowBufToNettyBuf(buf); ByteBuffer byteBuffer = nettyBuf.internalNioBuffer(4, 6); Assert.assertEquals(0, byteBuffer.position()); Assert.assertEquals(6, byteBuffer.limit()); @@ -77,7 +77,7 @@ public void testSetLEValues() { try (BufferAllocator allocator = new RootAllocator(128); ArrowBuf buf = allocator.buffer(20); ) { - NettyArrowBuf nettyBuf = buf.asNettyBuffer(); + NettyArrowBuf nettyBuf = NettyArrowBuf.arrowBufToNettyBuf(buf); int [] intVals = new int[] {Integer.MIN_VALUE, Short.MIN_VALUE - 1, Short.MIN_VALUE, 0 , Short.MAX_VALUE , Short.MAX_VALUE + 1, Integer.MAX_VALUE}; for (int intValue :intVals ) { @@ -103,7 +103,7 @@ public void testSetLEValues() { public void testSetCompositeBuffer() { try (BufferAllocator allocator = new RootAllocator(128); ArrowBuf buf = allocator.buffer(20); - NettyArrowBuf buf2 = allocator.buffer(20).asNettyBuffer(); + NettyArrowBuf buf2 = NettyArrowBuf.arrowBufToNettyBuf(allocator.buffer(20)); ) { CompositeByteBuf byteBufs = new CompositeByteBuf(new ArrowByteBufAllocator(allocator), true, 1); @@ -111,7 +111,7 @@ public void testSetCompositeBuffer() { buf2.setInt(0, expected); buf2.writerIndex(4); byteBufs.addComponent(true, buf2); - buf.asNettyBuffer().setBytes(0, byteBufs, 4); + NettyArrowBuf.arrowBufToNettyBuf(buf).setBytes(0, byteBufs, 4); int actual = buf.getInt(0); Assert.assertEquals(expected, actual); } @@ -126,12 +126,12 @@ public void testGetCompositeBuffer() { true, 1); int expected = 4; buf.setInt(0, expected); - NettyArrowBuf buf2 = allocator.buffer(20).asNettyBuffer(); + NettyArrowBuf buf2 = NettyArrowBuf.arrowBufToNettyBuf(allocator.buffer(20)); // composite buffers are a bit weird, need to jump hoops // to set capacity. byteBufs.addComponent(true, buf2); byteBufs.capacity(20); - buf.asNettyBuffer().getBytes(0, byteBufs, 4); + NettyArrowBuf.arrowBufToNettyBuf(buf).getBytes(0, byteBufs, 4); int actual = byteBufs.getInt(0); Assert.assertEquals(expected, actual); byteBufs.component(0).release(); diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestEndianness.java b/java/netty-memory/src/test/java/org/apache/arrow/memory/TestEndianness.java similarity index 92% rename from java/memory/src/test/java/org/apache/arrow/memory/TestEndianness.java rename to java/netty-memory/src/test/java/org/apache/arrow/memory/TestEndianness.java index 0d42bfabd2b..cf9b7828563 100644 --- a/java/memory/src/test/java/org/apache/arrow/memory/TestEndianness.java +++ b/java/netty-memory/src/test/java/org/apache/arrow/memory/TestEndianness.java @@ -22,13 +22,14 @@ import org.junit.Test; import io.netty.buffer.ByteBuf; +import io.netty.buffer.NettyArrowBuf; public class TestEndianness { @Test public void testLittleEndian() { final BufferAllocator a = new RootAllocator(10000); - final ByteBuf b = a.buffer(4).asNettyBuffer(); + final ByteBuf b = NettyArrowBuf.arrowBufToNettyBuf(a.buffer(4)); b.setInt(0, 35); assertEquals(b.getByte(0), 35); assertEquals(b.getByte(1), 0); diff --git a/java/performance/pom.xml b/java/performance/pom.xml index b2cdbdd01df..103990a0767 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -53,7 +53,7 @@ org.apache.arrow - arrow-memory + arrow-memory-netty ${project.version} diff --git a/java/pom.xml b/java/pom.xml index acbfdd449b9..7aff53b56ee 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -686,6 +686,7 @@ performance algorithm adapter/avro + netty-memory diff --git a/java/tools/pom.xml b/java/tools/pom.xml index f5248b8fb36..c01f30d8b4f 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -23,7 +23,7 @@ org.apache.arrow - arrow-memory + arrow-memory-netty ${project.version} diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 66026a506c4..8452b09e74f 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -29,7 +29,7 @@ org.apache.arrow - arrow-memory + arrow-memory-netty ${project.version}