Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HBASE-25465 Use javac --release option for supporting cross version c… #4164

Merged
merged 1 commit into from
Mar 12, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,20 @@
import java.util.List;
import java.util.Objects;
import java.util.PriorityQueue;

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;

import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;

import org.apache.hadoop.hbase.shaded.protobuf.generated.FilterProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.BytesBytesPair;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.UnsafeAvailChecker;

/**
* This is optimized version of a standard FuzzyRowFilter Filters data based on fuzzy row key.
Expand All @@ -57,7 +58,7 @@
*/
@InterfaceAudience.Public
public class FuzzyRowFilter extends FilterBase {
private static final boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned();
private static final boolean UNSAFE_UNALIGNED = HBasePlatformDependent.unaligned();
private List<Pair<byte[], byte[]>> fuzzyKeysData;
private boolean done = false;

Expand Down
4 changes: 4 additions & 0 deletions hbase-common/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,10 @@
<groupId>org.apache.hbase.thirdparty</groupId>
<artifactId>hbase-shaded-netty</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase.thirdparty</groupId>
<artifactId>hbase-unsafe</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.SingleByteBuff;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.hadoop.hbase.util.UnsafeAccess;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.nio.ch.DirectBuffer;

import org.apache.hbase.thirdparty.com.google.common.collect.Sets;

Expand Down Expand Up @@ -335,11 +335,8 @@ public ByteBuff allocate(int size) {
public void clean() {
while (!buffers.isEmpty()) {
ByteBuffer b = buffers.poll();
if (b instanceof DirectBuffer) {
DirectBuffer db = (DirectBuffer) b;
if (db.cleaner() != null) {
db.cleaner().clean();
}
if (b.isDirect()) {
UnsafeAccess.freeDirectBuffer(b);
}
}
this.usedBufCount.set(0);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,25 +23,22 @@
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;

import org.apache.hadoop.hbase.io.ByteBuffAllocator.Recycler;
import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.hadoop.hbase.util.ObjectIntPair;
import org.apache.hadoop.hbase.util.UnsafeAccess;
import org.apache.hadoop.hbase.util.UnsafeAvailChecker;
import org.apache.yetus.audience.InterfaceAudience;

import sun.nio.ch.DirectBuffer;

/**
* An implementation of ByteBuff where a single BB backs the BBI. This just acts as a wrapper over a
* normal BB - offheap or onheap
*/
@InterfaceAudience.Private
public class SingleByteBuff extends ByteBuff {

private static final boolean UNSAFE_AVAIL = UnsafeAvailChecker.isAvailable();
private static final boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned();
private static final boolean UNSAFE_AVAIL = HBasePlatformDependent.isUnsafeAvailable();
private static final boolean UNSAFE_UNALIGNED = HBasePlatformDependent.unaligned();

// Underlying BB
private final ByteBuffer buf;
Expand All @@ -65,7 +62,7 @@ public SingleByteBuff(Recycler recycler, ByteBuffer buf) {
this.unsafeOffset = UnsafeAccess.BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset();
this.unsafeRef = buf.array();
} else {
this.unsafeOffset = ((DirectBuffer) buf).address();
this.unsafeOffset = UnsafeAccess.directBufferAddress(buf);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we move all of UnsafeAccess into HBasePlatformDependent?

}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,24 +30,23 @@
import org.apache.hadoop.hbase.io.ByteBufferWriter;
import org.apache.hadoop.hbase.io.util.StreamUtils;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.WritableUtils;
import org.apache.yetus.audience.InterfaceAudience;
import sun.nio.ch.DirectBuffer;

/**
* Utility functions for working with byte buffers, such as reading/writing
* variable-length long numbers.
*/
@SuppressWarnings("restriction")
@InterfaceAudience.Private
public final class ByteBufferUtils {
// "Compressed integer" serialization helper constants.
public final static int VALUE_MASK = 0x7f;
public final static int NEXT_BIT_SHIFT = 7;
public final static int NEXT_BIT_MASK = 1 << 7;
final static boolean UNSAFE_AVAIL = UnsafeAvailChecker.isAvailable();
public final static boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned();
final static boolean UNSAFE_AVAIL = HBasePlatformDependent.isUnsafeAvailable();
public final static boolean UNSAFE_UNALIGNED = HBasePlatformDependent.unaligned();

private ByteBufferUtils() {
}
Expand Down Expand Up @@ -78,11 +77,10 @@ static class ComparerHolder {

static Comparer getBestComparer() {
try {
Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
Class<? extends Comparer> theClass =
Class.forName(UNSAFE_COMPARER_NAME).asSubclass(Comparer.class);

@SuppressWarnings("unchecked")
Comparer comparer = (Comparer) theClass.getConstructor().newInstance();
return comparer;
return theClass.getConstructor().newInstance();
} catch (Throwable t) { // ensure we really catch *everything*
return PureJavaComparer.INSTANCE;
}
Expand Down Expand Up @@ -137,7 +135,7 @@ public int compareTo(byte[] buf1, int o1, int l1, ByteBuffer buf2, int o2, int l
long offset2Adj;
Object refObj2 = null;
if (buf2.isDirect()) {
offset2Adj = o2 + ((DirectBuffer)buf2).address();
offset2Adj = o2 + UnsafeAccess.directBufferAddress(buf2);
} else {
offset2Adj = o2 + buf2.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
refObj2 = buf2.array();
Expand All @@ -151,13 +149,13 @@ public int compareTo(ByteBuffer buf1, int o1, int l1, ByteBuffer buf2, int o2, i
long offset1Adj, offset2Adj;
Object refObj1 = null, refObj2 = null;
if (buf1.isDirect()) {
offset1Adj = o1 + ((DirectBuffer) buf1).address();
offset1Adj = o1 + UnsafeAccess.directBufferAddress(buf1);
} else {
offset1Adj = o1 + buf1.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
refObj1 = buf1.array();
}
if (buf2.isDirect()) {
offset2Adj = o2 + ((DirectBuffer) buf2).address();
offset2Adj = o2 + UnsafeAccess.directBufferAddress(buf2);
} else {
offset2Adj = o2 + buf2.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
refObj2 = buf2.array();
Expand All @@ -175,12 +173,11 @@ static class ConverterHolder {

static Converter getBestConverter() {
try {
Class<?> theClass = Class.forName(UNSAFE_CONVERTER_NAME);
Class<? extends Converter> theClass =
Class.forName(UNSAFE_CONVERTER_NAME).asSubclass(Converter.class);

// yes, UnsafeComparer does implement Comparer<byte[]>
@SuppressWarnings("unchecked")
Converter converter = (Converter) theClass.getConstructor().newInstance();
return converter;
return theClass.getConstructor().newInstance();
} catch (Throwable t) { // ensure we really catch *everything*
return PureJavaConverter.INSTANCE;
}
Expand Down Expand Up @@ -922,8 +919,8 @@ static int compareToUnsafe(Object obj1, long o1, int l1, Object obj2, long o2, i
* 64-bit.
*/
for (i = 0; i < strideLimit; i += stride) {
long lw = UnsafeAccess.theUnsafe.getLong(obj1, o1 + (long) i);
long rw = UnsafeAccess.theUnsafe.getLong(obj2, o2 + (long) i);
long lw = HBasePlatformDependent.getLong(obj1, o1 + (long) i);
long rw = HBasePlatformDependent.getLong(obj2, o2 + (long) i);
if (lw != rw) {
if (!UnsafeAccess.LITTLE_ENDIAN) {
return ((lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE)) ? -1 : 1;
Expand All @@ -944,8 +941,8 @@ static int compareToUnsafe(Object obj1, long o1, int l1, Object obj2, long o2, i

// The epilogue to cover the last (minLength % stride) elements.
for (; i < minLength; i++) {
int il = (UnsafeAccess.theUnsafe.getByte(obj1, o1 + i) & 0xFF);
int ir = (UnsafeAccess.theUnsafe.getByte(obj2, o2 + i) & 0xFF);
int il = (HBasePlatformDependent.getByte(obj1, o1 + i) & 0xFF);
int ir = (HBasePlatformDependent.getByte(obj2, o2 + i) & 0xFF);
if (il != ir) {
return il - ir;
}
Expand Down
26 changes: 9 additions & 17 deletions hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.io.WritableUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.misc.Unsafe;

import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;

Expand All @@ -56,7 +56,6 @@
* comparisons, hash code generation, manufacturing keys for HashMaps or
* HashSets, and can be used as key in maps or trees.
*/
@SuppressWarnings("restriction")
@InterfaceAudience.Public
@edu.umd.cs.findbugs.annotations.SuppressWarnings(
value="EQ_CHECK_FOR_OPERAND_NOT_COMPATIBLE_WITH_THIS",
Expand Down Expand Up @@ -126,7 +125,7 @@ public class Bytes implements Comparable<Bytes> {
// SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?)
public static final int ESTIMATED_HEAP_TAX = 16;

static final boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned();
static final boolean UNSAFE_UNALIGNED = HBasePlatformDependent.unaligned();

/**
* Returns length of the byte array, returning 0 if the array is null.
Expand Down Expand Up @@ -1359,21 +1358,17 @@ int putShort(byte[] bytes, int offset, short val) {

protected static final class UnsafeConverter extends Converter {

static final Unsafe theUnsafe;

public UnsafeConverter() {}

static {
if (UNSAFE_UNALIGNED) {
theUnsafe = UnsafeAccess.theUnsafe;
} else {
if (!UNSAFE_UNALIGNED) {
// It doesn't matter what we throw;
// it's swallowed in getBestComparer().
throw new Error();
}

// sanity check - this should never fail
if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
if (HBasePlatformDependent.arrayIndexScale(byte[].class) != 1) {
throw new AssertionError();
}
}
Expand Down Expand Up @@ -1412,7 +1407,7 @@ int putShort(byte[] bytes, int offset, short val) {

/**
* Provides a lexicographical comparer implementation; either a Java
* implementation or a faster implementation based on {@link Unsafe}.
* implementation or a faster implementation based on {@code Unsafe}.
*
* <p>Uses reflection to gracefully fall back to the Java implementation if
* {@code Unsafe} isn't available.
Expand Down Expand Up @@ -1469,18 +1464,15 @@ public int compareTo(byte[] buffer1, int offset1, int length1,
enum UnsafeComparer implements Comparer<byte[]> {
INSTANCE;

static final Unsafe theUnsafe;
static {
if (UNSAFE_UNALIGNED) {
theUnsafe = UnsafeAccess.theUnsafe;
} else {
if (!UNSAFE_UNALIGNED) {
// It doesn't matter what we throw;
// it's swallowed in getBestComparer().
throw new Error();
}

// sanity check - this should never fail
if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
if (HBasePlatformDependent.arrayIndexScale(byte[].class) != 1) {
throw new AssertionError();
}
}
Expand Down Expand Up @@ -1518,8 +1510,8 @@ public int compareTo(byte[] buffer1, int offset1, int length1,
* than 4 bytes even on 32-bit. On the other hand, it is substantially faster on 64-bit.
*/
for (i = 0; i < strideLimit; i += stride) {
long lw = theUnsafe.getLong(buffer1, offset1Adj + i);
long rw = theUnsafe.getLong(buffer2, offset2Adj + i);
long lw = HBasePlatformDependent.getLong(buffer1, offset1Adj + i);
long rw = HBasePlatformDependent.getLong(buffer2, offset2Adj + i);
if (lw != rw) {
if(!UnsafeAccess.LITTLE_ENDIAN) {
return ((lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE)) ? -1 : 1;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import java.lang.reflect.Modifier;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;

import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -186,13 +186,19 @@ private static final class HeaderSize {
private byte a;
}

private static final int ARRAY_OBJECT_INDEX_SCALE =
HBasePlatformDependent.arrayIndexScale(Object[].class);

private static final int ARRAY_BYTE_INDEX_SCALE =
HBasePlatformDependent.arrayIndexScale(byte[].class);

public UnsafeLayout() {
}

@Override
int headerSize() {
try {
return (int) UnsafeAccess.theUnsafe.objectFieldOffset(
return (int) HBasePlatformDependent.objectFieldOffset(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is platform dependent so should be moved into HBasePlatformDependent. Perhaps all of ClassSize should go up in there but at least these methods involving object layout concerns can be replaced with external static helpers in the thirdparty module.

HeaderSize.class.getDeclaredField("a"));
} catch (NoSuchFieldException | SecurityException e) {
LOG.error(e.toString(), e);
Expand All @@ -202,29 +208,28 @@ int headerSize() {

@Override
int arrayHeaderSize() {
return UnsafeAccess.theUnsafe.arrayBaseOffset(byte[].class);
return HBasePlatformDependent.arrayBaseOffset(byte[].class);
}

@Override
@SuppressWarnings("static-access")
int oopSize() {
// Unsafe.addressSize() returns 8, even with CompressedOops. This is how many bytes each
// element is allocated in an Object[].
return UnsafeAccess.theUnsafe.ARRAY_OBJECT_INDEX_SCALE;
return ARRAY_OBJECT_INDEX_SCALE;
}

@Override
@SuppressWarnings("static-access")
long sizeOfByteArray(int len) {
return align(ARRAY + len * UnsafeAccess.theUnsafe.ARRAY_BYTE_INDEX_SCALE);
return align(ARRAY + len * ARRAY_BYTE_INDEX_SCALE);
}
}

private static MemoryLayout getMemoryLayout() {
// Have a safeguard in case Unsafe estimate is wrong. This is static context, there is
// no configuration, so we look at System property.
String enabled = System.getProperty("hbase.memorylayout.use.unsafe");
if (UnsafeAvailChecker.isAvailable() && (enabled == null || Boolean.parseBoolean(enabled))) {
if (HBasePlatformDependent.isUnsafeAvailable() &&
(enabled == null || Boolean.parseBoolean(enabled))) {
LOG.debug("Using Unsafe to estimate memory layout");
return new UnsafeLayout();
}
Expand Down
Loading