Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Minor follow-up optimizations #148

Merged
merged 7 commits into from
Mar 27, 2024
Merged

Minor follow-up optimizations #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2ce98c6
Make JIT to fold more constants in Tsavorite
PaulusParssinen Mar 26, 2024
3943373
Use BitOperations.Log2 in SectorAlignedBufferPool.Position
PaulusParssinen Mar 26, 2024
ced10b8
Use BitOperations.Log2 in LimitedFixedBufferPool.Position
PaulusParssinen Mar 26, 2024
d7facf2
Simplify MallocFixedPageSize further by removing ReturnPhysicalAddress
PaulusParssinen Mar 27, 2024
1ecba77
Merge branch 'main' into more-tsavorite-const-folding
PaulusParssinen Mar 27, 2024
19e1830
Merge branch 'main' into more-tsavorite-const-folding
badrishc Mar 27, 2024
06f2741
Merge branch 'main' into more-tsavorite-const-folding
badrishc Mar 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 5 additions & 16 deletions libs/common/Memory/LimitedFixedBufferPool.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System;
using System.Diagnostics;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Threading;
using Microsoft.Extensions.Logging;
Expand Down Expand Up @@ -150,29 +151,17 @@ public void Print()
[MethodImpl(MethodImplOptions.AggressiveInlining)]
int Position(int v)
{
if (v < minAllocationSize || !IsPowerOfTwo(v))
if (v < minAllocationSize || !BitOperations.IsPow2(v))
return -1;

v /= minAllocationSize;

if (v == 1) return 0;
v--;

int r = 0; // r will be lg(v)
while (true) // unroll for more speed...
{
v >>= 1;
if (v == 0) break;
r++;
}
if (r + 1 >= numLevels)
int level = BitOperations.Log2((uint)v - 1) + 1;
if (level >= numLevels)
return -1;
return r + 1;
}

static bool IsPowerOfTwo(long x)
{
return (x > 0) && ((x & (x - 1)) == 0);
return level;
}
}
}
10 changes: 3 additions & 7 deletions libs/storage/Tsavorite/cs/src/core/Allocator/GenericAllocator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,9 @@ internal sealed unsafe class GenericAllocator<Key, Value> : AllocatorBase<Key, V
private readonly int ObjectBlockSize = 100 * (1 << 20);
// Tail offsets per segment, in object log
public readonly long[] segmentOffsets;
// Record sizes
private readonly SerializerSettings<Key, Value> SerializerSettings;
private readonly bool keyBlittable = Utility.IsBlittable<Key>();
private readonly bool valueBlittable = Utility.IsBlittable<Value>();


// We do not support variable-length keys in GenericAllocator
// Record sizes. We do not support variable-length keys in GenericAllocator
internal static int KeySize => Unsafe.SizeOf<Key>();
internal static int ValueSize => Unsafe.SizeOf<Value>();
internal static int RecordSize => Unsafe.SizeOf<Record<Key, Value>>();
Expand All @@ -75,7 +71,7 @@ public GenericAllocator(LogSettings settings, SerializerSettings<Key, Value> ser

SerializerSettings = serializerSettings ?? new SerializerSettings<Key, Value>();

if ((!keyBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
if ((!Utility.IsBlittable<Key>()) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
{
#if DEBUG
if (typeof(Key) != typeof(byte[]) && typeof(Key) != typeof(string))
Expand All @@ -84,7 +80,7 @@ public GenericAllocator(LogSettings settings, SerializerSettings<Key, Value> ser
SerializerSettings.keySerializer = ObjectSerializer.Get<Key>();
}

if ((!valueBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
if ((!Utility.IsBlittable<Value>()) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
{
#if DEBUG
if (typeof(Value) != typeof(byte[]) && typeof(Value) != typeof(string))
Expand Down
121 changes: 38 additions & 83 deletions libs/storage/Tsavorite/cs/src/core/Allocator/MallocFixedPageSize.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,13 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;

namespace Tsavorite.core
{
internal class CountdownWrapper
internal sealed class CountdownWrapper
{
// Separate event for sync code and tcs for async code: Do not block on async code.
private readonly CountdownEvent syncEvent;
Expand Down Expand Up @@ -60,12 +59,10 @@ internal void Decrement()
/// Memory allocator for objects
/// </summary>
/// <typeparam name="T"></typeparam>
public class MallocFixedPageSize<T> : IDisposable
public sealed class MallocFixedPageSize<T> : IDisposable
{
private const bool ForceUnpinnedAllocation = false;

// We will never get an index of 0 from (Bulk)Allocate
internal const long kInvalidAllocationIndex = 0;
private const long InvalidAllocationIndex = 0;

private const int PageSizeBits = 16;
private const int PageSize = 1 << PageSizeBits;
Expand All @@ -76,21 +73,17 @@ public class MallocFixedPageSize<T> : IDisposable
private readonly T[][] values = new T[LevelSize][];
private readonly IntPtr[] pointers = new IntPtr[LevelSize];

private readonly int RecordSize;

private volatile int writeCacheLevel;

private volatile int count;

internal bool IsPinned => isPinned;
private readonly bool isPinned;

private const bool ReturnPhysicalAddress = false;
internal static int RecordSize => Unsafe.SizeOf<T>();
internal static bool IsBlittable => Utility.IsBlittable<T>();

private int checkpointCallbackCount;
private SemaphoreSlim checkpointSemaphore;

private ConcurrentQueue<long> freeList;
private readonly ConcurrentQueue<long> freeList;

readonly ILogger logger;

Expand All @@ -99,7 +92,7 @@ private enum AllocationMode { None, Single, Bulk };
private AllocationMode allocationMode;
#endif

const int sector_size = 512;
const int SectorSize = 512;

private int initialAllocation = 0;

Expand All @@ -113,17 +106,11 @@ public unsafe MallocFixedPageSize(ILogger logger = null)
{
this.logger = logger;
freeList = new ConcurrentQueue<long>();
if (ForceUnpinnedAllocation)
isPinned = false;
else
isPinned = Utility.IsBlittable<T>();
Debug.Assert(isPinned || !ReturnPhysicalAddress, "ReturnPhysicalAddress requires pinning");

values[0] = GC.AllocateArray<T>(PageSize + sector_size, isPinned);
if (isPinned)
values[0] = GC.AllocateArray<T>(PageSize + SectorSize, pinned: IsBlittable);
if (IsBlittable)
{
pointers[0] = (IntPtr)(((long)Unsafe.AsPointer(ref values[0][0]) + (sector_size - 1)) & ~(sector_size - 1));
RecordSize = Marshal.SizeOf(values[0][0]);
pointers[0] = (IntPtr)(((long)Unsafe.AsPointer(ref values[0][0]) + (SectorSize - 1)) & ~(SectorSize - 1));
}

#if !(CALLOC)
Expand All @@ -136,24 +123,22 @@ public unsafe MallocFixedPageSize(ILogger logger = null)
// Allocate one block so we never return a null pointer; this allocation is never freed.
// Use BulkAllocate so the caller can still do either BulkAllocate or single Allocate().
BulkAllocate();
initialAllocation = kAllocateChunkSize;
initialAllocation = AllocateChunkSize;
#if DEBUG
// Clear this for the next allocation.
this.allocationMode = AllocationMode.None;
allocationMode = AllocationMode.None;
#endif
}

/// <summary>
/// Get physical address -- for Pinned only
/// Get physical address -- for blittable objects only
/// </summary>
/// <param name="logicalAddress">The logicalAddress of the allocation. For BulkAllocate, this may be an address within the chunk size, to reference that particular record.</param>
/// <returns></returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public long GetPhysicalAddress(long logicalAddress)
{
Debug.Assert(isPinned, "GetPhysicalAddress requires pinning");
if (ReturnPhysicalAddress)
return logicalAddress;
Debug.Assert(IsBlittable, "GetPhysicalAddress requires the values to be blittable");
return (long)pointers[logicalAddress >> PageSizeBits] + (logicalAddress & PageSizeMask) * RecordSize;
}

Expand All @@ -165,10 +150,8 @@ public long GetPhysicalAddress(long logicalAddress)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe ref T Get(long index)
{
if (ReturnPhysicalAddress)
throw new TsavoriteException("Physical pointer returned by allocator: de-reference pointer to get records instead of calling Get");
Debug.Assert(index != kInvalidAllocationIndex, "Invalid allocation index");
if (isPinned)
Debug.Assert(index != InvalidAllocationIndex, "Invalid allocation index");
if (IsBlittable)
return ref Unsafe.AsRef<T>((byte*)(pointers[index >> PageSizeBits]) + (index & PageSizeMask) * RecordSize);
else
return ref values[index >> PageSizeBits][index & PageSizeMask];
Expand All @@ -182,10 +165,8 @@ public unsafe ref T Get(long index)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe void Set(long index, ref T value)
{
if (ReturnPhysicalAddress)
throw new TsavoriteException("Physical pointer returned by allocator: de-reference pointer to set records instead of calling Set (otherwise, set ForceUnpinnedAllocation to true)");
Debug.Assert(index != kInvalidAllocationIndex, "Invalid allocation index");
if (isPinned)
Debug.Assert(index != InvalidAllocationIndex, "Invalid allocation index");
if (IsBlittable)
Unsafe.AsRef<T>((byte*)(pointers[index >> PageSizeBits]) + (index & PageSizeMask) * RecordSize) = value;
else
values[index >> PageSizeBits][index & PageSizeMask] = value;
Expand All @@ -198,14 +179,12 @@ public unsafe void Set(long index, ref T value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Free(long pointer)
{
if (!ReturnPhysicalAddress)
Get(pointer) = default;
freeList.Enqueue(pointer);
}

internal int FreeListCount => freeList.Count; // For test

internal const int kAllocateChunkSize = 16; // internal for test
internal const int AllocateChunkSize = 16; // internal for test

/// <summary>
/// Allocate a block of size RecordSize * kAllocateChunkSize.
Expand All @@ -217,10 +196,10 @@ public void Free(long pointer)
public unsafe long BulkAllocate()
{
#if DEBUG
Debug.Assert(this.allocationMode != AllocationMode.Single, "Cannot mix Single and Bulk allocation modes");
this.allocationMode = AllocationMode.Bulk;
Debug.Assert(allocationMode != AllocationMode.Single, "Cannot mix Single and Bulk allocation modes");
allocationMode = AllocationMode.Bulk;
#endif
return InternalAllocate(kAllocateChunkSize);
return InternalAllocate(AllocateChunkSize);
}

/// <summary>
Expand All @@ -231,8 +210,8 @@ public unsafe long BulkAllocate()
public unsafe long Allocate()
{
#if DEBUG
Debug.Assert(this.allocationMode != AllocationMode.Bulk, "Cannot mix Single and Bulk allocation modes");
this.allocationMode = AllocationMode.Single;
Debug.Assert(allocationMode != AllocationMode.Bulk, "Cannot mix Single and Bulk allocation modes");
allocationMode = AllocationMode.Single;
#endif
return InternalAllocate(1);
}
Expand All @@ -253,9 +232,9 @@ private unsafe long InternalAllocate(int blockSize)
// If index 0, then allocate space for next level.
if (index == 0)
{
var tmp = GC.AllocateArray<T>(PageSize + sector_size, isPinned);
if (isPinned)
pointers[1] = (IntPtr)(((long)Unsafe.AsPointer(ref tmp[0]) + (sector_size - 1)) & ~(sector_size - 1));
var tmp = GC.AllocateArray<T>(PageSize + SectorSize, pinned: IsBlittable);
if (IsBlittable)
pointers[1] = (IntPtr)(((long)Unsafe.AsPointer(ref tmp[0]) + (SectorSize - 1)) & ~(SectorSize - 1));

#if !(CALLOC)
Array.Clear(tmp, 0, PageSize);
Expand All @@ -265,10 +244,7 @@ private unsafe long InternalAllocate(int blockSize)
}

// Return location.
if (ReturnPhysicalAddress)
return ((long)pointers[0]) + index * RecordSize;
else
return index;
return index;
}

// See if write cache contains corresponding array.
Expand All @@ -281,16 +257,10 @@ private unsafe long InternalAllocate(int blockSize)
if (cache == baseAddr)
{
// Return location.
if (ReturnPhysicalAddress)
return ((long)pointers[baseAddr]) + (long)offset * RecordSize;
else
return index;
return index;
}
}

// Write cache did not work, so get level information from index.
// int level = GetLevelFromIndex(index);

// Spin-wait until level has an allocated array.
var spinner = new SpinWait();
while (true)
Expand All @@ -313,9 +283,9 @@ private unsafe long InternalAllocate(int blockSize)
// Allocate for next page
int newBaseAddr = baseAddr + 1;

var tmp = GC.AllocateArray<T>(PageSize + sector_size, isPinned);
if (isPinned)
pointers[newBaseAddr] = (IntPtr)(((long)Unsafe.AsPointer(ref tmp[0]) + (sector_size - 1)) & ~(sector_size - 1));
var tmp = GC.AllocateArray<T>(PageSize + SectorSize, pinned: IsBlittable);
if (IsBlittable)
pointers[newBaseAddr] = (IntPtr)(((long)Unsafe.AsPointer(ref tmp[0]) + (SectorSize - 1)) & ~(SectorSize - 1));

#if !(CALLOC)
Array.Clear(tmp, 0, PageSize);
Expand All @@ -326,19 +296,13 @@ private unsafe long InternalAllocate(int blockSize)
}

// Return location.
if (ReturnPhysicalAddress)
return ((long)pointers[baseAddr]) + (long)offset * RecordSize;
else
return index;
return index;
}

/// <summary>
/// Dispose
/// </summary>
public void Dispose()
{
count = 0;
}
public void Dispose() => count = 0;


#region Checkpoint
Expand All @@ -347,10 +311,7 @@ public void Dispose()
/// Is checkpoint complete
/// </summary>
/// <returns></returns>
public bool IsCheckpointCompleted()
{
return checkpointCallbackCount == 0;
}
public bool IsCheckpointCompleted() => checkpointCallbackCount == 0;

/// <summary>
/// Is checkpoint completed
Expand Down Expand Up @@ -416,7 +377,7 @@ internal unsafe void BeginCheckpoint(IDevice device, ulong offset, out ulong num

Buffer.MemoryCopy((void*)pointers[i], result.mem.aligned_pointer, writeSize, writeSize);
int j = 0;
if (i == 0) j += kAllocateChunkSize * RecordSize;
if (i == 0) j += AllocateChunkSize * RecordSize;
for (; j < writeSize; j += sizeof(HashBucket))
{
skipReadCache((HashBucket*)(result.mem.aligned_pointer + j));
Expand Down Expand Up @@ -450,19 +411,13 @@ private unsafe void AsyncFlushCallback(uint errorCode, uint numBytes, object con
/// Max valid address
/// </summary>
/// <returns></returns>
public int GetMaxValidAddress()
{
return count;
}
public int GetMaxValidAddress() => count;

/// <summary>
/// Get page size
/// </summary>
/// <returns></returns>
public int GetPageSize()
{
return PageSize;
}
public int GetPageSize() => PageSize;
#endregion

#region Recover
Expand Down
Loading