From ca133fca5ed8dc3757f69a646637b04905cac88f Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Wed, 5 May 2021 23:07:34 -0700 Subject: [PATCH 1/5] [C#] Support reducing memory footprint of base hlog circular buffer. --- cs/samples/MemOnlyCache/CacheSizeTracker.cs | 21 ++--- cs/samples/MemOnlyCache/Program.cs | 2 +- cs/src/core/Allocator/AllocatorBase.cs | 25 ++++-- cs/src/core/Allocator/BlittableAllocator.cs | 31 ++++++++ cs/src/core/Allocator/GenericAllocator.cs | 20 +++++ cs/src/core/Allocator/PageUnit.cs | 18 +++++ .../Allocator/VarLenBlittableAllocator.cs | 36 ++++++++- cs/src/core/Index/FASTER/LogAccessor.cs | 4 +- cs/src/core/Utilities/OverflowPool.cs | 76 +++++++++++++++++++ 9 files changed, 210 insertions(+), 23 deletions(-) create mode 100644 cs/src/core/Allocator/PageUnit.cs create mode 100644 cs/src/core/Utilities/OverflowPool.cs diff --git a/cs/samples/MemOnlyCache/CacheSizeTracker.cs b/cs/samples/MemOnlyCache/CacheSizeTracker.cs index 70b1924bf..102af253d 100644 --- a/cs/samples/MemOnlyCache/CacheSizeTracker.cs +++ b/cs/samples/MemOnlyCache/CacheSizeTracker.cs @@ -13,7 +13,7 @@ namespace MemOnlyCache public class CacheSizeTracker : IObserver> { readonly FasterKV store; - long storeSize; + long storeHeapSize; /// /// Target size request for FASTER @@ -23,34 +23,29 @@ public class CacheSizeTracker : IObserver /// Total size (bytes) used by FASTER including index and log /// - public long TotalSizeBytes => storeSize + store.OverflowBucketCount * 64; + public long TotalSizeBytes => storeHeapSize + store.IndexSize * 64 + store.Log.MemorySizeBytes + (store.ReadCache != null ? store.ReadCache.MemorySizeBytes : 0) + store.OverflowBucketCount * 64; /// /// Class to track and update cache size /// /// FASTER store instance - /// Memory size (bits) used by FASTER log settings - /// Target memory size of FASTER in bytes - public CacheSizeTracker(FasterKV store, int memorySizeBits, long targetMemoryBytes = long.MaxValue) + /// Initial target memory size of FASTER in bytes + public CacheSizeTracker(FasterKV store, long targetMemoryBytes = long.MaxValue) { this.store = store; if (targetMemoryBytes < long.MaxValue) - { Console.WriteLine("**** Setting initial target memory: {0,11:N2}KB", targetMemoryBytes / 1024.0); - this.TargetSizeBytes = targetMemoryBytes; - } + this.TargetSizeBytes = targetMemoryBytes; - storeSize = store.IndexSize * 64 + store.Log.MemorySizeBytes; + Console.WriteLine("Index size: {0}", store.IndexSize * 64); + Console.WriteLine("Total store size: {0}", TotalSizeBytes); // Register subscriber to receive notifications of log evictions from memory store.Log.SubscribeEvictions(this); // Include the separate read cache, if enabled if (store.ReadCache != null) - { - storeSize += store.ReadCache.MemorySizeBytes; store.ReadCache.SubscribeEvictions(this); - } } /// @@ -72,7 +67,7 @@ public void SetTargetSizeBytes(long newTargetSize) /// Add to the tracked size of FASTER. This is called by IFunctions as well as the subscriber to evictions (OnNext) /// /// - public void AddTrackedSize(int size) => Interlocked.Add(ref storeSize, size); + public void AddTrackedSize(int size) => Interlocked.Add(ref storeHeapSize, size); /// /// Subscriber to pages as they are getting evicted from main memory diff --git a/cs/samples/MemOnlyCache/Program.cs b/cs/samples/MemOnlyCache/Program.cs index 2290cb570..24dc8f2bc 100644 --- a/cs/samples/MemOnlyCache/Program.cs +++ b/cs/samples/MemOnlyCache/Program.cs @@ -86,7 +86,7 @@ static void Main() var numBucketBits = (int)Math.Ceiling(Math.Log2(numRecords)); h = new FasterKV(1L << numBucketBits, logSettings, comparer: new CacheKey()); - sizeTracker = new CacheSizeTracker(h, logSettings.MemorySizeBits, targetSize); + sizeTracker = new CacheSizeTracker(h, targetSize); // Initially populate store PopulateStore(numRecords); diff --git a/cs/src/core/Allocator/AllocatorBase.cs b/cs/src/core/Allocator/AllocatorBase.cs index 34e460525..4e352ccd8 100644 --- a/cs/src/core/Allocator/AllocatorBase.cs +++ b/cs/src/core/Allocator/AllocatorBase.cs @@ -565,6 +565,9 @@ internal void WriteAsync(IntPtr alignedSourceAddress, ulong alignedDes /// Page number to be cleared /// Offset to clear from (if partial clear) internal abstract void ClearPage(long page, int offset = 0); + + internal abstract void FreePage(long page); + /// /// Write page (async) /// @@ -729,6 +732,11 @@ public AllocatorBase(LogSettings settings, IFasterEqualityComparer comparer AlignedPageSizeBytes = ((PageSize + (sectorSize - 1)) & ~(sectorSize - 1)); } + /// + /// Number of extra overflow pages allocated + /// + internal abstract int OverflowPageCount { get; } + /// /// Initialize allocator /// @@ -791,7 +799,6 @@ public virtual void Dispose() OnEvictionObserver?.OnCompleted(); } - /// /// How many pages do we leave empty in the in-memory buffer (between 0 and BufferSize-1) /// @@ -998,12 +1005,21 @@ public long TryAllocate(int numSlots = 1) return -1; // RETRY_NOW } + // Allocate this page, if needed + int thisPageIndex = (localTailPageOffset.Page + 1) % BufferSize; + if ((!IsAllocated(thisPageIndex))) + { + AllocatePage(thisPageIndex); + } + + /* // Allocate next page in advance, if needed int nextPageIndex = (localTailPageOffset.Page + 2) % BufferSize; if ((!IsAllocated(nextPageIndex))) { AllocatePage(nextPageIndex); } + */ localTailPageOffset.Page++; localTailPageOffset.Offset = numSlots; @@ -1235,11 +1251,8 @@ public void OnPagesClosed(long newSafeHeadAddress) int closePage = (int)(closePageAddress >> LogPageSizeBits); int closePageIndex = closePage % BufferSize; - - if (!IsAllocated(closePageIndex)) - AllocatePage(closePageIndex); - else - ClearPage(closePage); + + FreePage(closePage); Utility.MonotonicUpdate(ref PageStatusIndicator[closePageIndex].LastClosedUntilAddress, closePageAddress + PageSize, out _); ShiftClosedUntilAddress(); diff --git a/cs/src/core/Allocator/BlittableAllocator.cs b/cs/src/core/Allocator/BlittableAllocator.cs index 02ae2a5c4..47b8012fd 100644 --- a/cs/src/core/Allocator/BlittableAllocator.cs +++ b/cs/src/core/Allocator/BlittableAllocator.cs @@ -25,9 +25,13 @@ public unsafe sealed class BlittableAllocator : AllocatorBase overflowPagePool; + public BlittableAllocator(LogSettings settings, IFasterEqualityComparer comparer, Action evictCallback = null, LightEpoch epoch = null, Action flushCallback = null) : base(settings, comparer, evictCallback, epoch, flushCallback) { + overflowPagePool = new OverflowPool(4, p => p.handle.Free()); + values = new byte[BufferSize][]; handles = new GCHandle[BufferSize]; pointers = new long[BufferSize]; @@ -107,6 +111,7 @@ public override void Dispose() handles = null; pointers = null; values = null; + overflowPagePool.Dispose(); } public override AddressInfo* GetKeyAddressInfo(long physicalAddress) @@ -125,6 +130,14 @@ public override void Dispose() /// internal override void AllocatePage(int index) { + if (overflowPagePool.TryGet(out var item)) + { + handles[index] = item.handle; + pointers[index] = item.pointer; + values[index] = item.value; + return; + } + var adjustedSize = PageSize + 2 * sectorSize; byte[] tmp = new byte[adjustedSize]; Array.Clear(tmp, 0, adjustedSize); @@ -135,6 +148,8 @@ internal override void AllocatePage(int index) values[index] = tmp; } + internal override int OverflowPageCount => overflowPagePool.Count; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public override long GetPhysicalAddress(long logicalAddress) { @@ -208,6 +223,22 @@ internal override void ClearPage(long page, int offset) } } + internal override void FreePage(long page) + { + ClearPage(page, 0); + if (EmptyPageCount > 0) + { + int index = (int)(page % BufferSize); + overflowPagePool.TryAdd(new PageUnit { + handle = handles[index], + pointer = pointers[index], + value = values[index] }); + values[index] = null; + pointers[index] = 0; + handles[index] = default; + } + } + /// /// Delete in-memory portion of the log /// diff --git a/cs/src/core/Allocator/GenericAllocator.cs b/cs/src/core/Allocator/GenericAllocator.cs index 75af9368e..10603dd50 100644 --- a/cs/src/core/Allocator/GenericAllocator.cs +++ b/cs/src/core/Allocator/GenericAllocator.cs @@ -39,9 +39,13 @@ public unsafe sealed class GenericAllocator : AllocatorBase(); private readonly bool valueBlittable = Utility.IsBlittable(); + private readonly OverflowPool[]> overflowPagePool; + public GenericAllocator(LogSettings settings, SerializerSettings serializerSettings, IFasterEqualityComparer comparer, Action evictCallback = null, LightEpoch epoch = null, Action flushCallback = null) : base(settings, comparer, evictCallback, epoch, flushCallback) { + overflowPagePool = new OverflowPool[]>(4); + if (settings.ObjectLogDevice == null) { throw new FasterException("LogSettings.ObjectLogDevice needs to be specified (e.g., use Devices.CreateLogDevice, AzureStorageDevice, or NullDevice)"); @@ -75,6 +79,8 @@ public GenericAllocator(LogSettings settings, SerializerSettings ser } } + internal override int OverflowPageCount => overflowPagePool.Count; + public override void Initialize() { Initialize(recordSize); @@ -188,6 +194,7 @@ public override void Dispose() } values = null; } + overflowPagePool.Dispose(); base.Dispose(); } @@ -224,6 +231,9 @@ internal override void AllocatePage(int index) internal Record[] AllocatePage() { + if (overflowPagePool.TryGet(out var item)) + return item; + Record[] tmp; if (PageSize % recordSize == 0) tmp = new Record[PageSize / recordSize]; @@ -308,6 +318,16 @@ internal override void ClearPage(long page, int offset) } } + internal override void FreePage(long page) + { + ClearPage(page, 0); + if (EmptyPageCount > 0) + { + overflowPagePool.TryAdd(values[page % BufferSize]); + values[page % BufferSize] = default; + } + } + private void WriteAsync(long flushPage, ulong alignedDestinationAddress, uint numBytesToWrite, DeviceIOCompletionCallback callback, PageAsyncFlushResult asyncResult, IDevice device, IDevice objlogDevice, long intendedDestinationPage = -1, long[] localSegmentOffsets = null) diff --git a/cs/src/core/Allocator/PageUnit.cs b/cs/src/core/Allocator/PageUnit.cs new file mode 100644 index 000000000..93cf7ef35 --- /dev/null +++ b/cs/src/core/Allocator/PageUnit.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Runtime.InteropServices; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FASTER.core +{ + struct PageUnit + { + public byte[] value; + public GCHandle handle; + public long pointer; + } +} + + diff --git a/cs/src/core/Allocator/VarLenBlittableAllocator.cs b/cs/src/core/Allocator/VarLenBlittableAllocator.cs index d6c4af38d..518b291b1 100644 --- a/cs/src/core/Allocator/VarLenBlittableAllocator.cs +++ b/cs/src/core/Allocator/VarLenBlittableAllocator.cs @@ -26,9 +26,13 @@ public unsafe sealed class VariableLengthBlittableAllocator : Alloca internal readonly IVariableLengthStruct KeyLength; internal readonly IVariableLengthStruct ValueLength; + private readonly OverflowPool overflowPagePool; + public VariableLengthBlittableAllocator(LogSettings settings, VariableLengthStructSettings vlSettings, IFasterEqualityComparer comparer, Action evictCallback = null, LightEpoch epoch = null, Action flushCallback = null) : base(settings, comparer, evictCallback, epoch, flushCallback) { + overflowPagePool = new OverflowPool(4, p => p.handle.Free()); + values = new byte[BufferSize][]; handles = new GCHandle[BufferSize]; pointers = new long[BufferSize]; @@ -52,6 +56,8 @@ public VariableLengthBlittableAllocator(LogSettings settings, VariableLengthStru } } + internal override int OverflowPageCount => overflowPagePool.Count; + public override void Initialize() { Initialize(Constants.kFirstValidAddress); @@ -198,6 +204,8 @@ public override void Serialize(ref Value src, long physicalAddress) /// public override void Dispose() { + base.Dispose(); + if (values != null) { for (int i = 0; i < values.Length; i++) @@ -210,7 +218,7 @@ public override void Dispose() handles = null; pointers = null; values = null; - base.Dispose(); + overflowPagePool.Dispose(); } public override AddressInfo* GetKeyAddressInfo(long physicalAddress) @@ -229,6 +237,14 @@ public override void Dispose() /// internal override void AllocatePage(int index) { + if (overflowPagePool.TryGet(out var item)) + { + handles[index] = item.handle; + pointers[index] = item.pointer; + values[index] = item.value; + return; + } + var adjustedSize = PageSize + 2 * sectorSize; byte[] tmp = new byte[adjustedSize]; Array.Clear(tmp, 0, adjustedSize); @@ -312,6 +328,24 @@ internal override void ClearPage(long page, int offset) } } + internal override void FreePage(long page) + { + ClearPage(page, 0); + if (EmptyPageCount > 0) + { + int index = (int)(page % BufferSize); + overflowPagePool.TryAdd(new PageUnit + { + handle = handles[index], + pointer = pointers[index], + value = values[index] + }); + values[index] = null; + pointers[index] = 0; + handles[index] = default; + } + } + /// /// Delete in-memory portion of the log /// diff --git a/cs/src/core/Index/FASTER/LogAccessor.cs b/cs/src/core/Index/FASTER/LogAccessor.cs index 3dfda639a..d3887d8ce 100644 --- a/cs/src/core/Index/FASTER/LogAccessor.cs +++ b/cs/src/core/Index/FASTER/LogAccessor.cs @@ -79,8 +79,8 @@ public int EmptyPageCount /// /// Memory used by log (not including reference heap object sizes) /// - public long MemorySizeBytes => ((long)allocator.BufferSize) << allocator.LogPageSizeBits; - + public long MemorySizeBytes => ((long)(allocator.BufferSize - allocator.EmptyPageCount + allocator.OverflowPageCount)) << allocator.LogPageSizeBits; + /// /// Truncate the log until, but not including, untilAddress. Make sure address corresponds to record boundary if snapToPageStart is set to false. /// diff --git a/cs/src/core/Utilities/OverflowPool.cs b/cs/src/core/Utilities/OverflowPool.cs new file mode 100644 index 000000000..ee84a6520 --- /dev/null +++ b/cs/src/core/Utilities/OverflowPool.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Collections.Concurrent; + +namespace FASTER.core +{ + /// + /// Fixed size pool of overflow objects + /// + /// + internal class OverflowPool : IDisposable + { + readonly int size; + readonly ConcurrentQueue itemQueue; + readonly Action disposer; + + /// + /// Number of pages in pool + /// + public int Count => itemQueue.Count; + + bool disposed = false; + + /// + /// Constructor + /// + /// + /// + public OverflowPool(int size, Action disposer = null) + { + this.size = size; + this.itemQueue = new ConcurrentQueue(); + this.disposer = disposer ?? (e => { }); + } + + /// + /// Try get overflow item, if it exists + /// + /// + /// + public bool TryGet(out T item) + { + return itemQueue.TryDequeue(out item); + } + + /// + /// Try to add overflow item to pool + /// + /// + public bool TryAdd(T item) + { + if (itemQueue.Count < size && !disposed) + { + itemQueue.Enqueue(item); + return true; + } + else + { + disposer(item); + return false; + } + } + + /// + /// Dispose + /// + public void Dispose() + { + disposed = true; + while (itemQueue.TryDequeue(out var item)) + disposer(item); + } + } +} \ No newline at end of file From a8994ec98fca67f5c60774913406d644608ba556 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Thu, 6 May 2021 16:08:23 -0700 Subject: [PATCH 2/5] Update 23-fasterkv-tuning.md --- docs/_docs/23-fasterkv-tuning.md | 51 ++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/docs/_docs/23-fasterkv-tuning.md b/docs/_docs/23-fasterkv-tuning.md index 8bf02bb99..554632400 100644 --- a/docs/_docs/23-fasterkv-tuning.md +++ b/docs/_docs/23-fasterkv-tuning.md @@ -61,7 +61,8 @@ log growth unnecessarily. ## Computing Total FASTER Memory -The total memory footprint of FASTER can be computed as: +The total memory footprint of FASTER, with the exception of heap objects pointed to by +the log if any, can be computed as: ``` store.IndexSize * 64 + store.OverflowBucketCount * 64 + store.Log.MemorySizeBytes @@ -70,16 +71,33 @@ store.IndexSize * 64 + store.OverflowBucketCount * 64 + store.Log.MemorySizeByte If the read cache is enabled, add `store.ReadCache.MemorySizeBytes`. -## Managing Log Size with C# objects +## Managing Hybrid Log Memory Size -The FASTER in-memory portion consists of `store.Log.BufferSize` pages in a circular buffer. The -buffer size is equal to 2X, where X = `MemorySizeBits` - `PageSizeBits`. +The in-memory portion of the log consists of up to `store.Log.BufferSize` pages in a circular buffer. The +maximum buffer size is equal to 2X, where X = `MemorySizeBits` - `PageSizeBits`. Pages in +the buffer are allocated on demand as it fills up. Once filled, we do not incur any more allocations, +and the log takes up `store.Log.BufferSize` pages in total, each of size 2PageSizeBits. + +We can dynamically reduce the number of allocated pages in the circular buffer using a knob, called +`store.Log.EmptyPageCount`. This parameter indicates how many pages are kept empty in memory in the +circular buffer. This knob can vary between 0 (full buffer is used; the default) and `BufferSize-1` +(only the tail page is used). Thus, we can reduce the effective number of pages occupied and used +by the log in memory, and therefore the memory utilization. + +For example, suppose `PageSizeBits` is 20 (i.e., each page is 220 = 1MB) and `MemorySizeBits` is +30 (i.e., 230 = 1GB total memory. We have a `BufferSize` of 1024 (i.e., 230-20) pages +in memory, each of size 1MB. If we set `EmptyPageCount` to 512, we will instead have 1024-512=512 pages in +memory, for a total memory utilization of 512x1MB = 512MB, half of the full utilization of 1GB. + +As mentioned above, the actual log size (in bytes) can be queried at any time via `store.Log.MemorySizeBytes`. + +## Log Memory Size with C# Heap Objects When FASTER stores C# class key or value objects, the `GenericAllocator` is used for the -log (and read cache), and these buffer pages contain pointers to reference data which may take different -sizes, making the control of total memory footprint of FASTER difficult in this scenario. -This is because the total number of pointers in memory remains fixed, based on total -memory size. +log (and read cache), and these buffer pages contain only pointers to the objects. The +objects themselves may take up arbitrary sizes. This makes the control of total memory footprint +of FASTER difficult in this scenario, even though the total number of pointers in memory remains +fixed. For example, when `PageSizeBits` is 14 (i.e., each page is 214 = 16KB) and `MemorySizeBits` is 25 (i.e., 225 = 32MB total memory), we have a `BufferSize` of @@ -87,13 +105,16 @@ For example, when `PageSizeBits` is 14 (i.e., each page is 214 = 16KB as keys and values, since each record takes up 24 bytes (8 byte record header + 8 byte key pointer + 8 byte value pointer), the buffer stores a fixed number of 32M / 24 = ~1.39M key-value pairs. These could take up an arbitrary amount of total memory, depending on sizes of the -stored objects. - -FASTER has two capabilities to help manage state: - -1. One can accurately track the total memory used by FASTER, using a cache size [tracker](https://github.com/microsoft/FASTER/blob/master/cs/samples/MemOnlyCache/CacheSizeTracker.cs) that lets `IFunctions` notify it of record additions and deletions, and by subscribing to evictions from the head of the in-memory log. Details are in the [MemOnlySample](https://github.com/microsoft/FASTER/tree/master/cs/samples/MemOnlyCache) sample, where we show how to track FASTER's total memory usage (including the heap objects, log, hash table, and overflow buckets) very accurately. -2. In order to control the number of key-value pairs in memory dynamically, FASTER exposes a knob, called `store.Log.EmptyPageCount`, that indicates how many pages are kept empty in memory in the circular buffer. By adjusting this knob (default 0), we can reduce the effective number of pages that hold objects in memory (and therefore the overall memory utilization) below `BufferSize`. This knob can vary between 0 (full buffer is used) and `BufferSize-1` (only tail page is used). In the [MemOnlySample](https://github.com/microsoft/FASTER/tree/master/cs/samples/MemOnlyCache) sample, we allow the application to dynamically adjust the total memory utilization of FASTER, by exploiting this `EmptyPageCount` knob in the cache size [tracker](https://github.com/microsoft/FASTER/blob/master/cs/samples/MemOnlyCache/CacheSizeTracker.cs) module. This knob is only useful when C# class key or value types are used (`GenericAllocator`). - +stored objects. We can reduce the number of pages in memory as described above, but it will no longer +correspond exactly to the total log memory utilization. + +One can accurately track the total memory used by FASTER, including heap objects, using a cache size +tracker that lets `IFunctions` notify it of record additions and deletions, and by subscribing to +evictions from the head of the in-memory log. Details are shown in the [MemOnlySample](https://github.com/microsoft/FASTER/tree/master/cs/samples/MemOnlyCache) +sample, where we show how to implement such a cache size [tracker](https://github.com/microsoft/FASTER/blob/master/cs/samples/MemOnlyCache/CacheSizeTracker.cs) +to: +1. Track FASTER's total memory usage (including the heap objects, log, hash table, and overflow buckets) accurately. +2. Set a target memory usage and tune `EmptyPageCount` to achieve this target memory utilization. ## Configuring the Read Cache From 576efe630cf98dede98a1b2f0fe00fda28f40cf0 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Thu, 6 May 2021 19:49:13 -0700 Subject: [PATCH 3/5] updates --- cs/samples/MemOnlyCache/CacheSizeTracker.cs | 4 ++-- cs/src/core/Allocator/AllocatorBase.cs | 5 +++++ cs/src/core/Allocator/BlittableAllocator.cs | 3 +++ cs/src/core/Allocator/GenericAllocator.cs | 3 +++ .../core/Allocator/VarLenBlittableAllocator.cs | 3 +++ cs/src/core/Index/FASTER/LogAccessor.cs | 17 +++++++++++------ docs/_docs/23-fasterkv-tuning.md | 2 +- 7 files changed, 28 insertions(+), 9 deletions(-) diff --git a/cs/samples/MemOnlyCache/CacheSizeTracker.cs b/cs/samples/MemOnlyCache/CacheSizeTracker.cs index 102af253d..e3ffedbe8 100644 --- a/cs/samples/MemOnlyCache/CacheSizeTracker.cs +++ b/cs/samples/MemOnlyCache/CacheSizeTracker.cs @@ -85,9 +85,9 @@ public void OnNext(IFasterScanIterator iter) AddTrackedSize(-size); // Adjust empty page count to drive towards desired memory utilization - if (TotalSizeBytes > TargetSizeBytes) + if (TotalSizeBytes > TargetSizeBytes && store.Log.AllocatableMemorySizeBytes >= store.Log.MemorySizeBytes) store.Log.EmptyPageCount++; - else if (TotalSizeBytes < TargetSizeBytes) + else if (TotalSizeBytes < TargetSizeBytes && store.Log.AllocatableMemorySizeBytes <= store.Log.MemorySizeBytes) store.Log.EmptyPageCount--; } diff --git a/cs/src/core/Allocator/AllocatorBase.cs b/cs/src/core/Allocator/AllocatorBase.cs index 4e352ccd8..2003b0d27 100644 --- a/cs/src/core/Allocator/AllocatorBase.cs +++ b/cs/src/core/Allocator/AllocatorBase.cs @@ -799,6 +799,11 @@ public virtual void Dispose() OnEvictionObserver?.OnCompleted(); } + /// + /// Number of pages in circular buffer that are allocated + /// + public int AllocatedPageCount; + /// /// How many pages do we leave empty in the in-memory buffer (between 0 and BufferSize-1) /// diff --git a/cs/src/core/Allocator/BlittableAllocator.cs b/cs/src/core/Allocator/BlittableAllocator.cs index 47b8012fd..1bad1c9db 100644 --- a/cs/src/core/Allocator/BlittableAllocator.cs +++ b/cs/src/core/Allocator/BlittableAllocator.cs @@ -130,6 +130,8 @@ public override void Dispose() /// internal override void AllocatePage(int index) { + Interlocked.Increment(ref AllocatedPageCount); + if (overflowPagePool.TryGet(out var item)) { handles[index] = item.handle; @@ -236,6 +238,7 @@ internal override void FreePage(long page) values[index] = null; pointers[index] = 0; handles[index] = default; + Interlocked.Decrement(ref AllocatedPageCount); } } diff --git a/cs/src/core/Allocator/GenericAllocator.cs b/cs/src/core/Allocator/GenericAllocator.cs index 10603dd50..7f50cf5a5 100644 --- a/cs/src/core/Allocator/GenericAllocator.cs +++ b/cs/src/core/Allocator/GenericAllocator.cs @@ -231,6 +231,8 @@ internal override void AllocatePage(int index) internal Record[] AllocatePage() { + Interlocked.Increment(ref AllocatedPageCount); + if (overflowPagePool.TryGet(out var item)) return item; @@ -325,6 +327,7 @@ internal override void FreePage(long page) { overflowPagePool.TryAdd(values[page % BufferSize]); values[page % BufferSize] = default; + Interlocked.Decrement(ref AllocatedPageCount); } } diff --git a/cs/src/core/Allocator/VarLenBlittableAllocator.cs b/cs/src/core/Allocator/VarLenBlittableAllocator.cs index 518b291b1..f33df6430 100644 --- a/cs/src/core/Allocator/VarLenBlittableAllocator.cs +++ b/cs/src/core/Allocator/VarLenBlittableAllocator.cs @@ -237,6 +237,8 @@ public override void Dispose() /// internal override void AllocatePage(int index) { + Interlocked.Increment(ref AllocatedPageCount); + if (overflowPagePool.TryGet(out var item)) { handles[index] = item.handle; @@ -343,6 +345,7 @@ internal override void FreePage(long page) values[index] = null; pointers[index] = 0; handles[index] = default; + Interlocked.Decrement(ref AllocatedPageCount); } } diff --git a/cs/src/core/Index/FASTER/LogAccessor.cs b/cs/src/core/Index/FASTER/LogAccessor.cs index d3887d8ce..206bff15f 100644 --- a/cs/src/core/Index/FASTER/LogAccessor.cs +++ b/cs/src/core/Index/FASTER/LogAccessor.cs @@ -63,7 +63,7 @@ public LogAccessor(FasterKV fht, AllocatorBase allocator public int FixedRecordSize => allocator.GetFixedRecordSize(); /// - /// How many pages do we leave empty in the in-memory buffer (between 0 and BufferSize-1) + /// Number of pages left empty or unallocated in the in-memory buffer (between 0 and BufferSize-1) /// public int EmptyPageCount { @@ -72,15 +72,20 @@ public int EmptyPageCount } /// - /// Circular buffer size (in number of pages) + /// Total in-memory circular buffer capacity (in number of pages) /// - public int BufferSizePages => allocator.BufferSize; + public int BufferSize => allocator.BufferSize; /// - /// Memory used by log (not including reference heap object sizes) + /// Actual memory used by log (not including heap objects) /// - public long MemorySizeBytes => ((long)(allocator.BufferSize - allocator.EmptyPageCount + allocator.OverflowPageCount)) << allocator.LogPageSizeBits; - + public long MemorySizeBytes => ((long)(allocator.AllocatedPageCount + allocator.OverflowPageCount)) << allocator.LogPageSizeBits; + + /// + /// Memory allocatable on the log (not including heap objects) + /// + public long AllocatableMemorySizeBytes => ((long)(allocator.BufferSize - allocator.EmptyPageCount + allocator.OverflowPageCount)) << allocator.LogPageSizeBits; + /// /// Truncate the log until, but not including, untilAddress. Make sure address corresponds to record boundary if snapToPageStart is set to false. /// diff --git a/docs/_docs/23-fasterkv-tuning.md b/docs/_docs/23-fasterkv-tuning.md index 554632400..da0a68341 100644 --- a/docs/_docs/23-fasterkv-tuning.md +++ b/docs/_docs/23-fasterkv-tuning.md @@ -89,7 +89,7 @@ For example, suppose `PageSizeBits` is 20 (i.e., each page is 220 = 1 in memory, each of size 1MB. If we set `EmptyPageCount` to 512, we will instead have 1024-512=512 pages in memory, for a total memory utilization of 512x1MB = 512MB, half of the full utilization of 1GB. -As mentioned above, the actual log size (in bytes) can be queried at any time via `store.Log.MemorySizeBytes`. +As mentioned above, the actual current log size (in bytes) can be queried at any time via `store.Log.MemorySizeBytes`. ## Log Memory Size with C# Heap Objects From eed844b91d194fd2d98ee1cbb768e80af2ea56ab Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Mon, 10 May 2021 13:55:02 -0700 Subject: [PATCH 4/5] cleanup --- cs/src/core/Allocator/AllocatorBase.cs | 30 +++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/cs/src/core/Allocator/AllocatorBase.cs b/cs/src/core/Allocator/AllocatorBase.cs index 2003b0d27..78317fcd9 100644 --- a/cs/src/core/Allocator/AllocatorBase.cs +++ b/cs/src/core/Allocator/AllocatorBase.cs @@ -981,19 +981,21 @@ public long TryAllocate(int numSlots = 1) #region HANDLE PAGE OVERFLOW if (localTailPageOffset.Offset > PageSize) { + int pageIndex = localTailPageOffset.Page + 1; + // All overflow threads try to shift addresses - long shiftAddress = ((long)(localTailPageOffset.Page + 1)) << LogPageSizeBits; + long shiftAddress = ((long)pageIndex) << LogPageSizeBits; PageAlignedShiftReadOnlyAddress(shiftAddress); PageAlignedShiftHeadAddress(shiftAddress); if (offset > PageSize) { - if (NeedToWait(localTailPageOffset.Page + 1)) + if (NeedToWait(pageIndex)) return 0; // RETRY_LATER return -1; // RETRY_NOW } - if (NeedToWait(localTailPageOffset.Page + 1)) + if (NeedToWait(pageIndex)) { // Reset to end of page so that next attempt can retry localTailPageOffset.Offset = PageSize; @@ -1002,7 +1004,7 @@ public long TryAllocate(int numSlots = 1) } // The thread that "makes" the offset incorrect should allocate next page and set new tail - if (CannotAllocate(localTailPageOffset.Page + 1)) + if (CannotAllocate(pageIndex)) { // Reset to end of page so that next attempt can retry localTailPageOffset.Offset = PageSize; @@ -1011,20 +1013,12 @@ public long TryAllocate(int numSlots = 1) } // Allocate this page, if needed - int thisPageIndex = (localTailPageOffset.Page + 1) % BufferSize; - if ((!IsAllocated(thisPageIndex))) - { - AllocatePage(thisPageIndex); - } + if (!IsAllocated(pageIndex % BufferSize)) + AllocatePage(pageIndex % BufferSize); - /* // Allocate next page in advance, if needed - int nextPageIndex = (localTailPageOffset.Page + 2) % BufferSize; - if ((!IsAllocated(nextPageIndex))) - { - AllocatePage(nextPageIndex); - } - */ + if (!IsAllocated((pageIndex + 1) % BufferSize)) + AllocatePage((pageIndex + 1) % BufferSize); localTailPageOffset.Page++; localTailPageOffset.Offset = numSlots; @@ -1090,7 +1084,7 @@ public long TryAllocateRetryNow(int numSlots = 1) return logicalAddress; } - + private bool CannotAllocate(int page) => page >= BufferSize + (ClosedUntilAddress >> LogPageSizeBits); private bool NeedToWait(int page) => page >= BufferSize + (FlushedUntilAddress >> LogPageSizeBits); @@ -1256,7 +1250,7 @@ public void OnPagesClosed(long newSafeHeadAddress) int closePage = (int)(closePageAddress >> LogPageSizeBits); int closePageIndex = closePage % BufferSize; - + FreePage(closePage); Utility.MonotonicUpdate(ref PageStatusIndicator[closePageIndex].LastClosedUntilAddress, closePageAddress + PageSize, out _); From 37b129c129e5366251d2464948cecfc3d88c1084 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Wed, 12 May 2021 17:18:48 -0700 Subject: [PATCH 5/5] Add Log API to set page count with option to wait for address shift to complete. --- cs/src/core/Allocator/AllocatorBase.cs | 4 ++-- cs/src/core/Index/FASTER/LogAccessor.cs | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/cs/src/core/Allocator/AllocatorBase.cs b/cs/src/core/Allocator/AllocatorBase.cs index 78317fcd9..5503da2a1 100644 --- a/cs/src/core/Allocator/AllocatorBase.cs +++ b/cs/src/core/Allocator/AllocatorBase.cs @@ -111,7 +111,7 @@ public abstract partial class AllocatorBase : IDisposable /// /// HeadOFfset lag address /// - protected long HeadOffsetLagAddress; + internal long HeadOffsetLagAddress; /// /// Log mutable fraction @@ -830,7 +830,7 @@ public int EmptyPageCount } // Force eviction now if empty page count has increased - if (value > oldEPC) + if (value >= oldEPC) { bool prot = true; if (!epoch.ThisInstanceProtected()) diff --git a/cs/src/core/Index/FASTER/LogAccessor.cs b/cs/src/core/Index/FASTER/LogAccessor.cs index 206bff15f..6cf55e0ac 100644 --- a/cs/src/core/Index/FASTER/LogAccessor.cs +++ b/cs/src/core/Index/FASTER/LogAccessor.cs @@ -71,6 +71,21 @@ public int EmptyPageCount set { allocator.EmptyPageCount = value; } } + /// + /// Set empty page count in allocator + /// + /// New empty page count + /// Whether to wait for shift addresses to complete + public void SetEmptyPageCount(int pageCount, bool wait = false) + { + allocator.EmptyPageCount = pageCount; + if (wait) + { + long newHeadAddress = (allocator.GetTailAddress() & ~allocator.PageSizeMask) - allocator.HeadOffsetLagAddress; + ShiftHeadAddress(newHeadAddress, wait); + } + } + /// /// Total in-memory circular buffer capacity (in number of pages) ///