Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CASSANDRA-20190][trunk] MemoryUtil.setInt/getInt and similar use the wrong endianness #3936

Open
wants to merge 3 commits into
base: trunk
Choose a base branch
from
Open

[CASSANDRA-20190][trunk] MemoryUtil.setInt/getInt and similar use the wrong endianness #3936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a57094c
Add NativeEndianMemoryUtil, move all methods which uses (or should) n…
Feb 26, 2025
8bc30f5
Add tests for NativeEndianMemoryUtil.get/putXByByte
Feb 27, 2025
df598f5
Add LittleEndianMemoryUtil and use it in Memory
netudima Mar 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions src/java/org/apache/cassandra/db/NativeClustering.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.HeapCloner;
import org.apache.cassandra.utils.memory.MemoryUtil;
import org.apache.cassandra.utils.memory.NativeEndianMemoryUtil;
import org.apache.cassandra.utils.memory.NativeAllocator;

public class NativeClustering implements Clustering<ByteBuffer>
Expand All @@ -50,30 +51,30 @@ public NativeClustering(NativeAllocator allocator, OpOrder.Group writeOp, Cluste

peer = allocator.allocate(metadataSize + dataSize + bitmapSize, writeOp);
long bitmapStart = peer + metadataSize;
MemoryUtil.setShort(peer, (short) count);
MemoryUtil.setShort(peer + (metadataSize - 2), (short) dataSize); // goes at the end of the other offsets
NativeEndianMemoryUtil.setShort(peer, (short) count);
NativeEndianMemoryUtil.setShort(peer + (metadataSize - 2), (short) dataSize); // goes at the end of the other offsets

MemoryUtil.setByte(bitmapStart, bitmapSize, (byte) 0);
NativeEndianMemoryUtil.setByte(bitmapStart, bitmapSize, (byte) 0);
long dataStart = peer + metadataSize + bitmapSize;
int dataOffset = 0;
for (int i = 0 ; i < count ; i++)
{
MemoryUtil.setShort(peer + 2 + i * 2, (short) dataOffset);
NativeEndianMemoryUtil.setShort(peer + 2 + i * 2, (short) dataOffset);

ByteBuffer value = clustering.bufferAt(i);
if (value == null)
{
long boffset = bitmapStart + (i >>> 3);
int b = MemoryUtil.getByte(boffset);
int b = NativeEndianMemoryUtil.getByte(boffset);
b |= 1 << (i & 7);
MemoryUtil.setByte(boffset, (byte) b);
NativeEndianMemoryUtil.setByte(boffset, (byte) b);
continue;
}

assert value.order() == ByteOrder.BIG_ENDIAN;

int size = value.remaining();
MemoryUtil.setBytes(dataStart + dataOffset, value);
NativeEndianMemoryUtil.setBytes(dataStart + dataOffset, value);
dataOffset += size;
}
}
Expand All @@ -90,13 +91,13 @@ public ClusteringPrefix<ByteBuffer> clustering()

public int size()
{
return MemoryUtil.getShort(peer);
return NativeEndianMemoryUtil.getUnsignedShort(peer);
}

public int dataSize()
{
int dataSizeOffset = (size() * 2) + 2; // metadataSize - 2
return MemoryUtil.getShort(peer + dataSizeOffset);
return NativeEndianMemoryUtil.getUnsignedShort(peer + dataSizeOffset);
}

public ByteBuffer get(int i)
Expand All @@ -109,12 +110,12 @@ public ByteBuffer get(int i)
int metadataSize = (size * 2) + 4;
int bitmapSize = ((size + 7) >>> 3);
long bitmapStart = peer + metadataSize;
int b = MemoryUtil.getByte(bitmapStart + (i >>> 3));
int b = NativeEndianMemoryUtil.getByte(bitmapStart + (i >>> 3));
if ((b & (1 << (i & 7))) != 0)
return null;

int startOffset = MemoryUtil.getShort(peer + 2 + i * 2);
int endOffset = MemoryUtil.getShort(peer + 4 + i * 2);
int startOffset = NativeEndianMemoryUtil.getUnsignedShort(peer + 2 + i * 2);
int endOffset = NativeEndianMemoryUtil.getUnsignedShort(peer + 4 + i * 2);
return MemoryUtil.getByteBuffer(bitmapStart + bitmapSize + startOffset,
endOffset - startOffset,
ByteOrder.BIG_ENDIAN);
Expand Down
9 changes: 5 additions & 4 deletions src/java/org/apache/cassandra/db/NativeDecoratedKey.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.MemoryUtil;
import org.apache.cassandra.utils.memory.NativeAllocator;
import org.apache.cassandra.utils.memory.NativeEndianMemoryUtil;

public class NativeDecoratedKey extends DecoratedKey
{
Expand All @@ -39,7 +40,7 @@ public NativeDecoratedKey(Token token, NativeAllocator allocator, OpOrder.Group

int size = key.remaining();
this.peer = allocator.allocate(4 + size, writeOp);
MemoryUtil.setInt(peer, size);
NativeEndianMemoryUtil.setInt(peer, size);
MemoryUtil.setBytes(peer + 4, key);
}

Expand All @@ -50,14 +51,14 @@ public NativeDecoratedKey(Token token, NativeAllocator allocator, OpOrder.Group

int size = keyBytes.length;
this.peer = allocator.allocate(4 + size, writeOp);
MemoryUtil.setInt(peer, size);
NativeEndianMemoryUtil.setInt(peer, size);
MemoryUtil.setBytes(peer + 4, keyBytes, 0, size);
}

@Inline
int length()
{
return MemoryUtil.getInt(peer);
return NativeEndianMemoryUtil.getInt(peer);
}

@Inline
Expand All @@ -75,7 +76,7 @@ public ByteBuffer getKey()
@Override
public int getKeyLength()
{
return MemoryUtil.getInt(peer);
return NativeEndianMemoryUtil.getInt(peer);
}

@Override
Expand Down
33 changes: 17 additions & 16 deletions src/java/org/apache/cassandra/db/rows/NativeCell.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.MemoryUtil;
import org.apache.cassandra.utils.memory.NativeAllocator;
import org.apache.cassandra.utils.memory.NativeEndianMemoryUtil;

public class NativeCell extends AbstractCell<ByteBuffer>
{
Expand Down Expand Up @@ -101,11 +102,11 @@ public NativeCell(NativeAllocator allocator,

// cellpath? : timestamp : ttl : localDeletionTime : length : <data> : [cell path length] : [<cell path data>]
peer = allocator.allocate((int) size, writeOp);
MemoryUtil.setByte(peer + HAS_CELLPATH, (byte)(path == null ? 0 : 1));
MemoryUtil.setLong(peer + TIMESTAMP, timestamp);
MemoryUtil.setInt(peer + TTL, ttl);
MemoryUtil.setInt(peer + DELETION, localDeletionTimeUnsignedInteger);
MemoryUtil.setInt(peer + LENGTH, value.remaining());
NativeEndianMemoryUtil.setByte(peer + HAS_CELLPATH, (byte)(path == null ? 0 : 1));
NativeEndianMemoryUtil.setLong(peer + TIMESTAMP, timestamp);
NativeEndianMemoryUtil.setInt(peer + TTL, ttl);
NativeEndianMemoryUtil.setInt(peer + DELETION, localDeletionTimeUnsignedInteger);
NativeEndianMemoryUtil.setInt(peer + LENGTH, value.remaining());
MemoryUtil.setBytes(peer + VALUE, value);

if (path != null)
Expand All @@ -114,7 +115,7 @@ public NativeCell(NativeAllocator allocator,
assert pathbuffer.order() == ByteOrder.BIG_ENDIAN;

long offset = peer + VALUE + value.remaining();
MemoryUtil.setInt(offset, pathbuffer.remaining());
NativeEndianMemoryUtil.setInt(offset, pathbuffer.remaining());
MemoryUtil.setBytes(offset + 4, pathbuffer);
}
}
Expand All @@ -126,17 +127,17 @@ private static long offHeapSizeWithoutPath(int length)

public long timestamp()
{
return MemoryUtil.getLong(peer + TIMESTAMP);
return NativeEndianMemoryUtil.getLong(peer + TIMESTAMP);
}

public int ttl()
{
return MemoryUtil.getInt(peer + TTL);
return NativeEndianMemoryUtil.getInt(peer + TTL);
}

public ByteBuffer value()// FIXME: add native accessor
{
int length = MemoryUtil.getInt(peer + LENGTH);
int length = NativeEndianMemoryUtil.getInt(peer + LENGTH);
return MemoryUtil.getByteBuffer(peer + VALUE, length, ByteOrder.BIG_ENDIAN);
}

Expand All @@ -147,16 +148,16 @@ public ValueAccessor<ByteBuffer> accessor()

public int valueSize()
{
return MemoryUtil.getInt(peer + LENGTH);
return NativeEndianMemoryUtil.getInt(peer + LENGTH);
}

public CellPath path()
{
if (!hasPath())
return null;

long offset = peer + VALUE + MemoryUtil.getInt(peer + LENGTH);
int size = MemoryUtil.getInt(offset);
long offset = peer + VALUE + NativeEndianMemoryUtil.getInt(peer + LENGTH);
int size = NativeEndianMemoryUtil.getInt(offset);
return CellPath.create(MemoryUtil.getByteBuffer(offset + 4, size, ByteOrder.BIG_ENDIAN));
}

Expand Down Expand Up @@ -194,20 +195,20 @@ public long unsharedHeapSizeExcludingData()

public long offHeapSize()
{
long size = offHeapSizeWithoutPath(MemoryUtil.getInt(peer + LENGTH));
long size = offHeapSizeWithoutPath(NativeEndianMemoryUtil.getInt(peer + LENGTH));
if (hasPath())
size += 4 + MemoryUtil.getInt(peer + size);
size += 4 + NativeEndianMemoryUtil.getInt(peer + size);
return size;
}

private boolean hasPath()
{
return MemoryUtil.getByte(peer+ HAS_CELLPATH) != 0;
return NativeEndianMemoryUtil.getByte(peer + HAS_CELLPATH) != 0;
}

@Override
protected int localDeletionTimeAsUnsignedInt()
{
return MemoryUtil.getInt(peer + DELETION);
return NativeEndianMemoryUtil.getInt(peer + DELETION);
}
}
Loading