Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions java/adapter/jdbc/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,6 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</dependency>

</dependencies>

</project>
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,10 @@
import java.sql.SQLException;

import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.util.MemoryUtil;
import org.apache.arrow.vector.BitVectorHelper;
import org.apache.arrow.vector.VarCharVector;

import io.netty.util.internal.PlatformDependent;

/**
* Consumer which consume clob type values from {@link ResultSet}.
* Write the data to {@link org.apache.arrow.vector.VarCharVector}.
Expand Down Expand Up @@ -97,8 +96,12 @@ public void consume(ResultSet resultSet) throws SQLException {
while ((dataBuffer.writerIndex() + bytes.length) > dataBuffer.capacity()) {
vector.reallocDataBuffer();
}
PlatformDependent.copyMemory(bytes, 0,
dataBuffer.memoryAddress() + startIndex + totalBytes, bytes.length);
MemoryUtil.UNSAFE.copyMemory(
bytes,
MemoryUtil.BYTE_ARRAY_BASE_OFFSET,
null,
dataBuffer.memoryAddress() + startIndex + totalBytes,
bytes.length);

totalBytes += bytes.length;
read += readSize;
Expand Down Expand Up @@ -144,8 +147,12 @@ public void consume(ResultSet resultSet) throws SQLException {
while ((dataBuffer.writerIndex() + bytes.length) > dataBuffer.capacity()) {
vector.reallocDataBuffer();
}
PlatformDependent.copyMemory(bytes, 0,
dataBuffer.memoryAddress() + startIndex + totalBytes, bytes.length);
MemoryUtil.UNSAFE.copyMemory(
bytes,
MemoryUtil.BYTE_ARRAY_BASE_OFFSET,
null,
dataBuffer.memoryAddress() + startIndex + totalBytes,
bytes.length);

totalBytes += bytes.length;
read += readSize;
Expand Down
4 changes: 0 additions & 4 deletions java/algorithm/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,6 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</dependency>
</dependencies>

<build>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@
package org.apache.arrow.algorithm.sort;

import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.util.MemoryUtil;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.BaseFixedWidthVector;
import org.apache.arrow.vector.BitVectorHelper;
import org.apache.arrow.vector.IntVector;

import io.netty.util.internal.PlatformDependent;

/**
* Default out-of-place sorter for fixed-width vectors.
* It is an out-of-place sort, with time complexity O(n*log(n)).
Expand Down Expand Up @@ -69,7 +68,7 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator<V> co
BitVectorHelper.unsetBit(dstValidityBuffer, dstIndex);
} else {
BitVectorHelper.setBit(dstValidityBuffer, dstIndex);
PlatformDependent.copyMemory(
MemoryUtil.UNSAFE.copyMemory(
srcValueBuffer.memoryAddress() + srcIndex * valueWidth,
dstValueBuffer.memoryAddress() + dstIndex * valueWidth,
valueWidth);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@
package org.apache.arrow.algorithm.sort;

import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.util.MemoryUtil;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.BaseVariableWidthVector;
import org.apache.arrow.vector.BitVectorHelper;
import org.apache.arrow.vector.IntVector;

import io.netty.util.internal.PlatformDependent;

/**
* Default sorter for variable-width vectors.
* It is an out-of-place sort, with time complexity O(n*log(n)).
Expand Down Expand Up @@ -80,7 +79,7 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator<V> co
BitVectorHelper.setBit(dstValidityBuffer, dstIndex);
int srcOffset = srcOffsetBuffer.getInt(srcIndex * BaseVariableWidthVector.OFFSET_WIDTH);
int valueLength = srcOffsetBuffer.getInt((srcIndex + 1) * BaseVariableWidthVector.OFFSET_WIDTH) - srcOffset;
PlatformDependent.copyMemory(
MemoryUtil.UNSAFE.copyMemory(
srcValueBuffer.memoryAddress() + srcOffset,
dstValueBuffer.memoryAddress() + dstOffset,
valueLength);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.memory.util.MemoryUtil;
import org.apache.arrow.memory.util.hash.ArrowBufHasher;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.VectorVisitor;
Expand All @@ -37,8 +38,6 @@
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.apache.arrow.vector.util.TransferPair;

import io.netty.util.internal.PlatformDependent;

/**
* BaseFixedWidthVector provides an abstract interface for
* implementing vectors of fixed width values. The vectors are nullable
Expand Down Expand Up @@ -859,7 +858,7 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) {
BitVectorHelper.unsetBit(this.getValidityBuffer(), thisIndex);
} else {
BitVectorHelper.setBit(this.getValidityBuffer(), thisIndex);
PlatformDependent.copyMemory(from.getDataBuffer().memoryAddress() + (long) fromIndex * typeWidth,
MemoryUtil.UNSAFE.copyMemory(from.getDataBuffer().memoryAddress() + (long) fromIndex * typeWidth,
this.getDataBuffer().memoryAddress() + (long) thisIndex * typeWidth, typeWidth);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,15 @@

package org.apache.arrow.vector;

import static io.netty.util.internal.PlatformDependent.getByte;
import static io.netty.util.internal.PlatformDependent.getInt;
import static io.netty.util.internal.PlatformDependent.getLong;
import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt;

import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.BoundsChecking;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.util.MemoryUtil;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.util.DataSizeRoundingUtil;

import io.netty.util.internal.PlatformDependent;

/**
* Helper class for performing generic operations on a bit vector buffer.
Expand Down Expand Up @@ -260,23 +257,23 @@ public static boolean checkAllBitsEqualTo(

int index = 0;
while (index + 8 <= fullBytesCount) {
long longValue = getLong(validityBuffer.memoryAddress() + index);
long longValue = MemoryUtil.UNSAFE.getLong(validityBuffer.memoryAddress() + index);
if (longValue != (long) intToCompare) {
return false;
}
index += 8;
}

if (index + 4 <= fullBytesCount) {
int intValue = getInt(validityBuffer.memoryAddress() + index);
int intValue = MemoryUtil.UNSAFE.getInt(validityBuffer.memoryAddress() + index);
if (intValue != intToCompare) {
return false;
}
index += 4;
}

while (index < fullBytesCount) {
byte byteValue = getByte(validityBuffer.memoryAddress() + index);
byte byteValue = MemoryUtil.UNSAFE.getByte(validityBuffer.memoryAddress() + index);
if (byteValue != (byte) intToCompare) {
return false;
}
Expand All @@ -285,7 +282,7 @@ public static boolean checkAllBitsEqualTo(

// handling with the last bits
if (remainder != 0) {
byte byteValue = getByte(validityBuffer.memoryAddress() + sizeInBytes - 1);
byte byteValue = MemoryUtil.UNSAFE.getByte(validityBuffer.memoryAddress() + sizeInBytes - 1);
byte mask = (byte) ((1 << remainder) - 1);
byteValue = (byte) (byteValue & mask);
if (checkOneBits) {
Expand Down Expand Up @@ -395,13 +392,13 @@ public static void concatBits(ArrowBuf input1, int numBits1, ArrowBuf input2, in

// copy the first bit set
if (input1 != output) {
PlatformDependent.copyMemory(input1.memoryAddress(), output.memoryAddress(), numBytes1);
MemoryUtil.UNSAFE.copyMemory(input1.memoryAddress(), output.memoryAddress(), numBytes1);
}

if (bitIndex(numBits1) == 0) {
// The number of bits for the first bit set is a multiple of 8, so the boundary is at byte boundary.
// For this case, we have a shortcut to copy all bytes from the second set after the byte boundary.
PlatformDependent.copyMemory(input2.memoryAddress(), output.memoryAddress() + numBytes1, numBytes2);
MemoryUtil.UNSAFE.copyMemory(input2.memoryAddress(), output.memoryAddress() + numBytes1, numBytes2);
return;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.util.MemoryUtil;
import org.apache.arrow.vector.complex.impl.Decimal256ReaderImpl;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.holders.Decimal256Holder;
Expand All @@ -35,7 +36,6 @@
import org.apache.arrow.vector.util.DecimalUtility;
import org.apache.arrow.vector.util.TransferPair;

import io.netty.util.internal.PlatformDependent;

/**
* Decimal256Vector implements a fixed width vector (32 bytes) of
Expand Down Expand Up @@ -222,13 +222,13 @@ public void setBigEndian(int index, byte[] value) {

long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH;
if (length == 0) {
PlatformDependent.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH, (byte) 0);
MemoryUtil.UNSAFE.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH, (byte) 0);
return;
}
if (LITTLE_ENDIAN) {
// swap bytes to convert BE to LE
for (int byteIdx = 0; byteIdx < length; ++byteIdx) {
PlatformDependent.putByte(outAddress + byteIdx, value[length - 1 - byteIdx]);
MemoryUtil.UNSAFE.putByte(outAddress + byteIdx, value[length - 1 - byteIdx]);
}

if (length == TYPE_WIDTH) {
Expand All @@ -238,16 +238,21 @@ public void setBigEndian(int index, byte[] value) {
if (length < TYPE_WIDTH) {
// sign extend
final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00);
PlatformDependent.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad);
MemoryUtil.UNSAFE.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad);
return;
}
} else {
if (length <= TYPE_WIDTH) {
// copy data from value to outAddress
PlatformDependent.copyMemory(value, 0, outAddress + Decimal256Vector.TYPE_WIDTH - length, length);
MemoryUtil.UNSAFE.copyMemory(
value,
MemoryUtil.BYTE_ARRAY_BASE_OFFSET,
null,
outAddress + Decimal256Vector.TYPE_WIDTH - length,
length);
// sign extend
final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00);
PlatformDependent.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad);
MemoryUtil.UNSAFE.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad);
return;
}
}
Expand Down Expand Up @@ -285,20 +290,20 @@ public void setSafe(int index, long start, ArrowBuf buffer, int length) {
long inAddress = buffer.memoryAddress() + start;
long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH;
if (LITTLE_ENDIAN) {
PlatformDependent.copyMemory(inAddress, outAddress, length);
MemoryUtil.UNSAFE.copyMemory(inAddress, outAddress, length);
// sign extend
if (length < TYPE_WIDTH) {
byte msb = PlatformDependent.getByte(inAddress + length - 1);
byte msb = MemoryUtil.UNSAFE.getByte(inAddress + length - 1);
final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
PlatformDependent.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad);
MemoryUtil.UNSAFE.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad);
}
} else {
PlatformDependent.copyMemory(inAddress, outAddress + Decimal256Vector.TYPE_WIDTH - length, length);
MemoryUtil.UNSAFE.copyMemory(inAddress, outAddress + Decimal256Vector.TYPE_WIDTH - length, length);
// sign extend
if (length < TYPE_WIDTH) {
byte msb = PlatformDependent.getByte(inAddress);
byte msb = MemoryUtil.UNSAFE.getByte(inAddress);
final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
PlatformDependent.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad);
MemoryUtil.UNSAFE.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad);
}
}
}
Expand All @@ -325,22 +330,22 @@ public void setBigEndianSafe(int index, long start, ArrowBuf buffer, int length)
if (LITTLE_ENDIAN) {
// swap bytes to convert BE to LE
for (int byteIdx = 0; byteIdx < length; ++byteIdx) {
byte val = PlatformDependent.getByte((inAddress + length - 1) - byteIdx);
PlatformDependent.putByte(outAddress + byteIdx, val);
byte val = MemoryUtil.UNSAFE.getByte((inAddress + length - 1) - byteIdx);
MemoryUtil.UNSAFE.putByte(outAddress + byteIdx, val);
}
// sign extend
if (length < 32) {
byte msb = PlatformDependent.getByte(inAddress);
byte msb = MemoryUtil.UNSAFE.getByte(inAddress);
final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
PlatformDependent.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad);
MemoryUtil.UNSAFE.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad);
}
} else {
PlatformDependent.copyMemory(inAddress, outAddress + Decimal256Vector.TYPE_WIDTH - length, length);
MemoryUtil.UNSAFE.copyMemory(inAddress, outAddress + Decimal256Vector.TYPE_WIDTH - length, length);
// sign extend
if (length < TYPE_WIDTH) {
byte msb = PlatformDependent.getByte(inAddress);
byte msb = MemoryUtil.UNSAFE.getByte(inAddress);
final byte pad = (byte) (msb < 0 ? 0xFF : 0x00);
PlatformDependent.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad);
MemoryUtil.UNSAFE.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad);
}
}
}
Expand Down
Loading