Skip to content

[CASSANDRA-20190][5.0] MemoryUtil.setInt/getInt and similar use the wrong endianness #4093

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
+851 −363
Closed

[CASSANDRA-20190][5.0] MemoryUtil.setInt/getInt and similar use the wrong endianness #4093

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions src/java/org/apache/cassandra/db/NativeClustering.java
View file
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.HeapCloner;
import org.apache.cassandra.utils.memory.MemoryUtil;
import org.apache.cassandra.utils.memory.NativeEndianMemoryUtil;
import org.apache.cassandra.utils.memory.NativeAllocator;

public class NativeClustering implements Clustering<ByteBuffer>
@@ -50,30 +51,30 @@ public NativeClustering(NativeAllocator allocator, OpOrder.Group writeOp, Cluste

peer = allocator.allocate(metadataSize + dataSize + bitmapSize, writeOp);
long bitmapStart = peer + metadataSize;
MemoryUtil.setShort(peer, (short) count);
MemoryUtil.setShort(peer + (metadataSize - 2), (short) dataSize); // goes at the end of the other offsets
NativeEndianMemoryUtil.setShort(peer, (short) count);
NativeEndianMemoryUtil.setShort(peer + (metadataSize - 2), (short) dataSize); // goes at the end of the other offsets

MemoryUtil.setByte(bitmapStart, bitmapSize, (byte) 0);
NativeEndianMemoryUtil.setByte(bitmapStart, bitmapSize, (byte) 0);
long dataStart = peer + metadataSize + bitmapSize;
int dataOffset = 0;
for (int i = 0 ; i < count ; i++)
{
MemoryUtil.setShort(peer + 2 + i * 2, (short) dataOffset);
NativeEndianMemoryUtil.setShort(peer + 2 + i * 2, (short) dataOffset);

ByteBuffer value = clustering.bufferAt(i);
if (value == null)
{
long boffset = bitmapStart + (i >>> 3);
int b = MemoryUtil.getByte(boffset);
int b = NativeEndianMemoryUtil.getByte(boffset);
b |= 1 << (i & 7);
MemoryUtil.setByte(boffset, (byte) b);
NativeEndianMemoryUtil.setByte(boffset, (byte) b);
continue;
}

assert value.order() == ByteOrder.BIG_ENDIAN;

int size = value.remaining();
MemoryUtil.setBytes(dataStart + dataOffset, value);
NativeEndianMemoryUtil.setBytes(dataStart + dataOffset, value);
dataOffset += size;
}
}
@@ -90,13 +91,13 @@ public ClusteringPrefix<ByteBuffer> clustering()

public int size()
{
return MemoryUtil.getShort(peer);
return NativeEndianMemoryUtil.getUnsignedShort(peer);
}

public int dataSize()
{
int dataSizeOffset = (size() * 2) + 2; // metadataSize - 2
return MemoryUtil.getShort(peer + dataSizeOffset);
return NativeEndianMemoryUtil.getUnsignedShort(peer + dataSizeOffset);
}

public ByteBuffer get(int i)
@@ -109,12 +110,12 @@ public ByteBuffer get(int i)
int metadataSize = (size * 2) + 4;
int bitmapSize = ((size + 7) >>> 3);
long bitmapStart = peer + metadataSize;
int b = MemoryUtil.getByte(bitmapStart + (i >>> 3));
int b = NativeEndianMemoryUtil.getByte(bitmapStart + (i >>> 3));
if ((b & (1 << (i & 7))) != 0)
return null;

int startOffset = MemoryUtil.getShort(peer + 2 + i * 2);
int endOffset = MemoryUtil.getShort(peer + 4 + i * 2);
int startOffset = NativeEndianMemoryUtil.getUnsignedShort(peer + 2 + i * 2);
int endOffset = NativeEndianMemoryUtil.getUnsignedShort(peer + 4 + i * 2);
return MemoryUtil.getByteBuffer(bitmapStart + bitmapSize + startOffset,
endOffset - startOffset,
ByteOrder.BIG_ENDIAN);
9 changes: 5 additions & 4 deletions src/java/org/apache/cassandra/db/NativeDecoratedKey.java
View file
Original file line number Diff line number Diff line change
@@ -26,6 +26,7 @@
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.MemoryUtil;
import org.apache.cassandra.utils.memory.NativeAllocator;
import org.apache.cassandra.utils.memory.NativeEndianMemoryUtil;

public class NativeDecoratedKey extends DecoratedKey
{
@@ -39,7 +40,7 @@ public NativeDecoratedKey(Token token, NativeAllocator allocator, OpOrder.Group

int size = key.remaining();
this.peer = allocator.allocate(4 + size, writeOp);
MemoryUtil.setInt(peer, size);
NativeEndianMemoryUtil.setInt(peer, size);
MemoryUtil.setBytes(peer + 4, key);
}

@@ -50,14 +51,14 @@ public NativeDecoratedKey(Token token, NativeAllocator allocator, OpOrder.Group

int size = keyBytes.length;
this.peer = allocator.allocate(4 + size, writeOp);
MemoryUtil.setInt(peer, size);
NativeEndianMemoryUtil.setInt(peer, size);
MemoryUtil.setBytes(peer + 4, keyBytes, 0, size);
}

@Inline
int length()
{
return MemoryUtil.getInt(peer);
return NativeEndianMemoryUtil.getInt(peer);
}

@Inline
@@ -75,7 +76,7 @@ public ByteBuffer getKey()
@Override
public int getKeyLength()
{
return MemoryUtil.getInt(peer);
return NativeEndianMemoryUtil.getInt(peer);
}

@Override
33 changes: 17 additions & 16 deletions src/java/org/apache/cassandra/db/rows/NativeCell.java
View file
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.MemoryUtil;
import org.apache.cassandra.utils.memory.NativeAllocator;
import org.apache.cassandra.utils.memory.NativeEndianMemoryUtil;

public class NativeCell extends AbstractCell<ByteBuffer>
{
@@ -101,11 +102,11 @@ public NativeCell(NativeAllocator allocator,

// cellpath? : timestamp : ttl : localDeletionTime : length : <data> : [cell path length] : [<cell path data>]
peer = allocator.allocate((int) size, writeOp);
MemoryUtil.setByte(peer + HAS_CELLPATH, (byte)(path == null ? 0 : 1));
MemoryUtil.setLong(peer + TIMESTAMP, timestamp);
MemoryUtil.setInt(peer + TTL, ttl);
MemoryUtil.setInt(peer + DELETION, localDeletionTimeUnsignedInteger);
MemoryUtil.setInt(peer + LENGTH, value.remaining());
NativeEndianMemoryUtil.setByte(peer + HAS_CELLPATH, (byte)(path == null ? 0 : 1));
NativeEndianMemoryUtil.setLong(peer + TIMESTAMP, timestamp);
NativeEndianMemoryUtil.setInt(peer + TTL, ttl);
NativeEndianMemoryUtil.setInt(peer + DELETION, localDeletionTimeUnsignedInteger);
NativeEndianMemoryUtil.setInt(peer + LENGTH, value.remaining());
MemoryUtil.setBytes(peer + VALUE, value);

if (path != null)
@@ -114,7 +115,7 @@ public NativeCell(NativeAllocator allocator,
assert pathbuffer.order() == ByteOrder.BIG_ENDIAN;

long offset = peer + VALUE + value.remaining();
MemoryUtil.setInt(offset, pathbuffer.remaining());
NativeEndianMemoryUtil.setInt(offset, pathbuffer.remaining());
MemoryUtil.setBytes(offset + 4, pathbuffer);
}
}
@@ -126,17 +127,17 @@ private static long offHeapSizeWithoutPath(int length)

public long timestamp()
{
return MemoryUtil.getLong(peer + TIMESTAMP);
return NativeEndianMemoryUtil.getLong(peer + TIMESTAMP);
}

public int ttl()
{
return MemoryUtil.getInt(peer + TTL);
return NativeEndianMemoryUtil.getInt(peer + TTL);
}

public ByteBuffer value()// FIXME: add native accessor
{
int length = MemoryUtil.getInt(peer + LENGTH);
int length = NativeEndianMemoryUtil.getInt(peer + LENGTH);
return MemoryUtil.getByteBuffer(peer + VALUE, length, ByteOrder.BIG_ENDIAN);
}

@@ -147,16 +148,16 @@ public ValueAccessor<ByteBuffer> accessor()

public int valueSize()
{
return MemoryUtil.getInt(peer + LENGTH);
return NativeEndianMemoryUtil.getInt(peer + LENGTH);
}

public CellPath path()
{
if (!hasPath())
return null;

long offset = peer + VALUE + MemoryUtil.getInt(peer + LENGTH);
int size = MemoryUtil.getInt(offset);
long offset = peer + VALUE + NativeEndianMemoryUtil.getInt(peer + LENGTH);
int size = NativeEndianMemoryUtil.getInt(offset);
return CellPath.create(MemoryUtil.getByteBuffer(offset + 4, size, ByteOrder.BIG_ENDIAN));
}

@@ -194,20 +195,20 @@ public long unsharedHeapSizeExcludingData()

public long offHeapSize()
{
long size = offHeapSizeWithoutPath(MemoryUtil.getInt(peer + LENGTH));
long size = offHeapSizeWithoutPath(NativeEndianMemoryUtil.getInt(peer + LENGTH));
if (hasPath())
size += 4 + MemoryUtil.getInt(peer + size);
size += 4 + NativeEndianMemoryUtil.getInt(peer + size);
return size;
}

private boolean hasPath()
{
return MemoryUtil.getByte(peer+ HAS_CELLPATH) != 0;
return NativeEndianMemoryUtil.getByte(peer + HAS_CELLPATH) != 0;
}

@Override
protected int localDeletionTimeAsUnsignedInt()
{
return MemoryUtil.getInt(peer + DELETION);
return NativeEndianMemoryUtil.getInt(peer + DELETION);
}
}
12 changes: 12 additions & 0 deletions src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummary.java
View file
Original file line number Diff line number Diff line change
@@ -454,6 +454,18 @@ public <T extends InputStream & DataInputPlus> IndexSummary deserialize(T in, IP
entries.free();
throw ioe;
}

// Before 5.0 offsets were written using Native Endian, now they are stored as Little Endian,
// so we apply a heuristic here to detect
// if the loading index summary was created on a Big Endian machine using Native Endian format
if (offsets.size() > 0)
{
int offset = offsets.getInt(0);
int offsetReversed = Integer.reverseBytes(offset);
if (offsetReversed > 0 && offset > offsetReversed || offset - offsets.size() < 0)
throw new IOException(String.format("Rebuilding index summary because offset value (%d) at position: %d " +
"is Big Endian while Little Endian is expected", offset, 0));
}
// our on-disk representation treats the offsets and the summary data as one contiguous structure,
// in which the offsets are based from the start of the structure. i.e., if the offsets occupy
// X bytes, the value of the first offset will be X. In memory we split the two regions up, so that
Loading