From 09445f37b622db10150b05cde4aa66e49ab5f2eb Mon Sep 17 00:00:00 2001 From: Ted Hart <15467143+TedHartMS@users.noreply.github.com> Date: Mon, 7 Jun 2021 23:14:22 -0700 Subject: [PATCH] add FASTER.benchmark cmdline options: --sd --sm --noaff --chkptms --dumpdist (#485) --- cs/benchmark/ConcurrentDictionaryBenchmark.cs | 24 +++++----- cs/benchmark/FasterSpanByteYcsbBenchmark.cs | 45 ++++++++++--------- cs/benchmark/FasterYcsbBenchmark.cs | 45 ++++++++++--------- cs/benchmark/Options.cs | 36 ++++++++++++++- cs/benchmark/TestLoader.cs | 32 ++++++++----- cs/benchmark/YcsbConstants.cs | 20 --------- cs/benchmark/scripts/compare_runs.ps1 | 25 ++++++++++- 7 files changed, 140 insertions(+), 87 deletions(-) diff --git a/cs/benchmark/ConcurrentDictionaryBenchmark.cs b/cs/benchmark/ConcurrentDictionaryBenchmark.cs index 7dcafc6ad..ef301f470 100644 --- a/cs/benchmark/ConcurrentDictionaryBenchmark.cs +++ b/cs/benchmark/ConcurrentDictionaryBenchmark.cs @@ -27,6 +27,7 @@ internal class KeyComparer : IEqualityComparer internal unsafe class ConcurrentDictionary_YcsbBenchmark { + readonly TestLoader testLoader; readonly int numaStyle; readonly string distribution; readonly int readPercent; @@ -44,6 +45,7 @@ internal unsafe class ConcurrentDictionary_YcsbBenchmark internal ConcurrentDictionary_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLoader testLoader) { + this.testLoader = testLoader; init_keys_ = i_keys_; txn_keys_ = t_keys_; numaStyle = testLoader.Options.NumaStyle; @@ -67,7 +69,7 @@ internal ConcurrentDictionary_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLo for (int i = 0; i < 8; i++) input_[i].value = i; - store = new ConcurrentDictionary(testLoader.Options.ThreadCount, YcsbConstants.kMaxKey, new KeyComparer()); + store = new ConcurrentDictionary(testLoader.Options.ThreadCount, testLoader.MaxKey, new KeyComparer()); } internal void Dispose() @@ -101,9 +103,9 @@ private void RunYcsb(int thread_idx) while (!done) { long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; - while (chunk_idx >= YcsbConstants.kTxnCount) + while (chunk_idx >= testLoader.TxnCount) { - if (chunk_idx == YcsbConstants.kTxnCount) + if (chunk_idx == testLoader.TxnCount) idx_ = 0; chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; } @@ -205,7 +207,7 @@ internal unsafe (double, double) Run(TestLoader testLoader) } sw.Stop(); - double insertsPerSecond = ((double)YcsbConstants.kInitCount / sw.ElapsedMilliseconds) * 1000; + double insertsPerSecond = ((double)testLoader.InitCount / sw.ElapsedMilliseconds) * 1000; Console.WriteLine(TestStats.GetLoadingTimeLine(insertsPerSecond, sw.ElapsedMilliseconds)); idx_ = 0; @@ -227,7 +229,7 @@ internal unsafe (double, double) Run(TestLoader testLoader) Stopwatch swatch = new Stopwatch(); swatch.Start(); - if (YcsbConstants.kPeriodicCheckpointMilliseconds <= 0) + if (testLoader.Options.PeriodicCheckpointMilliseconds <= 0) { Thread.Sleep(TimeSpan.FromSeconds(testLoader.Options.RunSeconds)); } @@ -236,8 +238,8 @@ internal unsafe (double, double) Run(TestLoader testLoader) double runSeconds = 0; while (runSeconds < testLoader.Options.RunSeconds) { - Thread.Sleep(TimeSpan.FromMilliseconds(YcsbConstants.kPeriodicCheckpointMilliseconds)); - runSeconds += YcsbConstants.kPeriodicCheckpointMilliseconds / 1000; + Thread.Sleep(TimeSpan.FromMilliseconds(testLoader.Options.PeriodicCheckpointMilliseconds)); + runSeconds += testLoader.Options.PeriodicCheckpointMilliseconds / 1000; } } @@ -282,7 +284,7 @@ private void SetupYcsb(int thread_idx) Value value = default; for (long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; - chunk_idx < YcsbConstants.kInitCount; + chunk_idx < testLoader.InitCount; chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize) { for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize; ++idx) @@ -381,10 +383,10 @@ void DoContinuousMeasurements() #region Load Data - internal static void CreateKeyVectors(out Key[] i_keys, out Key[] t_keys) + internal static void CreateKeyVectors(TestLoader testLoader, out Key[] i_keys, out Key[] t_keys) { - i_keys = new Key[YcsbConstants.kInitCount]; - t_keys = new Key[YcsbConstants.kTxnCount]; + i_keys = new Key[testLoader.InitCount]; + t_keys = new Key[testLoader.TxnCount]; } internal class KeySetter : IKeySetter { diff --git a/cs/benchmark/FasterSpanByteYcsbBenchmark.cs b/cs/benchmark/FasterSpanByteYcsbBenchmark.cs index 46afe2a4d..124a7ccea 100644 --- a/cs/benchmark/FasterSpanByteYcsbBenchmark.cs +++ b/cs/benchmark/FasterSpanByteYcsbBenchmark.cs @@ -16,9 +16,10 @@ namespace FASTER.benchmark internal class FasterSpanByteYcsbBenchmark { // Ensure sizes are aligned to chunk sizes - const long kInitCount = YcsbConstants.kChunkSize * (YcsbConstants.kInitCount / YcsbConstants.kChunkSize); - const long kTxnCount = YcsbConstants.kChunkSize * (YcsbConstants.kTxnCount / YcsbConstants.kChunkSize); + static long InitCount; + static long TxnCount; + readonly TestLoader testLoader; readonly ManualResetEventSlim waiter = new ManualResetEventSlim(); readonly int numaStyle; readonly int readPercent; @@ -46,6 +47,7 @@ internal FasterSpanByteYcsbBenchmark(KeySpanByte[] i_keys_, KeySpanByte[] t_keys (testLoader.Options.NumaStyle == 1 && testLoader.Options.ThreadCount <= numGrps * (numProcs - 1))) Native32.AffinitizeThreadRoundRobin(numProcs - 1); + this.testLoader = testLoader; init_keys_ = i_keys_; txn_keys_ = t_keys_; numaStyle = testLoader.Options.NumaStyle; @@ -73,13 +75,13 @@ internal FasterSpanByteYcsbBenchmark(KeySpanByte[] i_keys_, KeySpanByte[] t_keys device = Devices.CreateLogDevice(TestLoader.DevicePath, preallocateFile: true, deleteOnClose: true); - if (YcsbConstants.kSmallMemoryLog) + if (testLoader.Options.UseSmallMemoryLog) store = new FasterKV - (YcsbConstants.kMaxKey / 2, new LogSettings { LogDevice = device, PreallocateLog = true, PageSizeBits = 22, SegmentSizeBits = 26, MemorySizeBits = 26 }, + (testLoader.MaxKey / 2, new LogSettings { LogDevice = device, PreallocateLog = true, PageSizeBits = 22, SegmentSizeBits = 26, MemorySizeBits = 26 }, new CheckpointSettings { CheckPointType = CheckpointType.Snapshot, CheckpointDir = testLoader.BackupPath }); else store = new FasterKV - (YcsbConstants.kMaxKey / 2, new LogSettings { LogDevice = device, PreallocateLog = true, MemorySizeBits = 35 }, + (testLoader.MaxKey / 2, new LogSettings { LogDevice = device, PreallocateLog = true, MemorySizeBits = 35 }, new CheckpointSettings { CheckPointType = CheckpointType.Snapshot, CheckpointDir = testLoader.BackupPath }); } @@ -121,14 +123,14 @@ private void RunYcsb(int thread_idx) int count = 0; #endif - var session = store.For(functions).NewSession(null, YcsbConstants.kAffinitizedSession); + var session = store.For(functions).NewSession(null, !testLoader.Options.NoThreadAffinity); while (!done) { long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; - while (chunk_idx >= kTxnCount) + while (chunk_idx >= TxnCount) { - if (chunk_idx == kTxnCount) + if (chunk_idx == TxnCount) idx_ = 0; chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; } @@ -146,7 +148,7 @@ private void RunYcsb(int thread_idx) if (idx % 512 == 0) { - if (YcsbConstants.kAffinitizedSession) + if (!testLoader.Options.NoThreadAffinity) session.Refresh(); session.CompletePending(false); } @@ -246,7 +248,7 @@ internal unsafe (double, double) Run(TestLoader testLoader) elapsedMs = sw.ElapsedMilliseconds; } - double insertsPerSecond = elapsedMs == 0 ? 0 : ((double)kInitCount / elapsedMs) * 1000; + double insertsPerSecond = elapsedMs == 0 ? 0 : ((double)InitCount / elapsedMs) * 1000; Console.WriteLine(TestStats.GetLoadingTimeLine(insertsPerSecond, elapsedMs)); Console.WriteLine(TestStats.GetAddressesLine(AddressLineNum.Before, store.Log.BeginAddress, store.Log.HeadAddress, store.Log.ReadOnlyAddress, store.Log.TailAddress)); @@ -258,11 +260,11 @@ internal unsafe (double, double) Run(TestLoader testLoader) idx_ = 0; - if (YcsbConstants.kDumpDistribution) + if (testLoader.Options.DumpDistribution) Console.WriteLine(store.DumpDistribution()); // Ensure first fold-over checkpoint is fast - if (YcsbConstants.kPeriodicCheckpointMilliseconds > 0 && YcsbConstants.kPeriodicCheckpointType == CheckpointType.FoldOver) + if (testLoader.Options.PeriodicCheckpointMilliseconds > 0 && testLoader.Options.PeriodicCheckpointType == CheckpointType.FoldOver) store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, true); Console.WriteLine("Executing experiment."); @@ -283,7 +285,7 @@ internal unsafe (double, double) Run(TestLoader testLoader) Stopwatch swatch = new Stopwatch(); swatch.Start(); - if (YcsbConstants.kPeriodicCheckpointMilliseconds <= 0) + if (testLoader.Options.PeriodicCheckpointMilliseconds <= 0) { Thread.Sleep(TimeSpan.FromSeconds(testLoader.Options.RunSeconds)); } @@ -292,10 +294,10 @@ internal unsafe (double, double) Run(TestLoader testLoader) var checkpointTaken = 0; while (swatch.ElapsedMilliseconds < 1000 * testLoader.Options.RunSeconds) { - if (checkpointTaken < swatch.ElapsedMilliseconds / YcsbConstants.kPeriodicCheckpointMilliseconds) + if (checkpointTaken < swatch.ElapsedMilliseconds / testLoader.Options.PeriodicCheckpointMilliseconds) { long start = swatch.ElapsedTicks; - if (store.TakeHybridLogCheckpoint(out _, YcsbConstants.kPeriodicCheckpointType, YcsbConstants.kPeriodicCheckpointTryIncremental)) + if (store.TakeHybridLogCheckpoint(out _, testLoader.Options.PeriodicCheckpointType, testLoader.Options.PeriodicCheckpointTryIncremental)) { store.CompleteCheckpointAsync().GetAwaiter().GetResult(); var timeTaken = (swatch.ElapsedTicks - start) / TimeSpan.TicksPerMillisecond; @@ -339,7 +341,7 @@ private void SetupYcsb(int thread_idx) waiter.Wait(); - var session = store.For(functions).NewSession(null, YcsbConstants.kAffinitizedSession); + var session = store.For(functions).NewSession(null, !testLoader.Options.NoThreadAffinity); #if DASHBOARD var tstart = Stopwatch.GetTimestamp(); @@ -352,7 +354,7 @@ private void SetupYcsb(int thread_idx) ref SpanByte _value = ref SpanByte.Reinterpret(value); for (long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; - chunk_idx < kInitCount; + chunk_idx < InitCount; chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize) { for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize; ++idx) @@ -455,10 +457,13 @@ void DoContinuousMeasurements() #region Load Data - internal static void CreateKeyVectors(out KeySpanByte[] i_keys, out KeySpanByte[] t_keys) + internal static void CreateKeyVectors(TestLoader testLoader, out KeySpanByte[] i_keys, out KeySpanByte[] t_keys) { - i_keys = new KeySpanByte[kInitCount]; - t_keys = new KeySpanByte[kTxnCount]; + InitCount = YcsbConstants.kChunkSize * (testLoader.InitCount / YcsbConstants.kChunkSize); + TxnCount = YcsbConstants.kChunkSize * (testLoader.TxnCount / YcsbConstants.kChunkSize); + + i_keys = new KeySpanByte[InitCount]; + t_keys = new KeySpanByte[TxnCount]; } internal class KeySetter : IKeySetter diff --git a/cs/benchmark/FasterYcsbBenchmark.cs b/cs/benchmark/FasterYcsbBenchmark.cs index b5d20df26..da68c15fb 100644 --- a/cs/benchmark/FasterYcsbBenchmark.cs +++ b/cs/benchmark/FasterYcsbBenchmark.cs @@ -16,9 +16,10 @@ namespace FASTER.benchmark internal class FASTER_YcsbBenchmark { // Ensure sizes are aligned to chunk sizes - const long kInitCount = YcsbConstants.kChunkSize * (YcsbConstants.kInitCount / YcsbConstants.kChunkSize); - const long kTxnCount = YcsbConstants.kChunkSize * (YcsbConstants.kTxnCount / YcsbConstants.kChunkSize); + static long InitCount; + static long TxnCount; + readonly TestLoader testLoader; readonly ManualResetEventSlim waiter = new ManualResetEventSlim(); readonly int numaStyle; readonly int readPercent; @@ -43,6 +44,7 @@ internal FASTER_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLoader testLoade (testLoader.Options.NumaStyle == 1 && testLoader.Options.ThreadCount <= numGrps * (numProcs - 1))) Native32.AffinitizeThreadRoundRobin(numProcs - 1); + this.testLoader = testLoader; init_keys_ = i_keys_; txn_keys_ = t_keys_; numaStyle = testLoader.Options.NumaStyle; @@ -73,13 +75,13 @@ internal FASTER_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLoader testLoade if (testLoader.Options.ThreadCount >= 16) device.ThrottleLimit = testLoader.Options.ThreadCount * 12; - if (YcsbConstants.kSmallMemoryLog) + if (testLoader.Options.UseSmallMemoryLog) store = new FasterKV - (YcsbConstants.kMaxKey / 4, new LogSettings { LogDevice = device, PreallocateLog = true, PageSizeBits = 25, SegmentSizeBits = 30, MemorySizeBits = 28 }, + (testLoader.MaxKey / 4, new LogSettings { LogDevice = device, PreallocateLog = true, PageSizeBits = 25, SegmentSizeBits = 30, MemorySizeBits = 28 }, new CheckpointSettings { CheckPointType = CheckpointType.Snapshot, CheckpointDir = testLoader.BackupPath }); else store = new FasterKV - (YcsbConstants.kMaxKey / 2, new LogSettings { LogDevice = device, PreallocateLog = true }, + (testLoader.MaxKey / 2, new LogSettings { LogDevice = device, PreallocateLog = true }, new CheckpointSettings { CheckPointType = CheckpointType.Snapshot, CheckpointDir = testLoader.BackupPath }); } @@ -117,14 +119,14 @@ private void RunYcsb(int thread_idx) int count = 0; #endif - var session = store.For(functions).NewSession(null, YcsbConstants.kAffinitizedSession); + var session = store.For(functions).NewSession(null, !testLoader.Options.NoThreadAffinity); while (!done) { long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; - while (chunk_idx >= kTxnCount) + while (chunk_idx >= TxnCount) { - if (chunk_idx == kTxnCount) + if (chunk_idx == TxnCount) idx_ = 0; chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; } @@ -142,7 +144,7 @@ private void RunYcsb(int thread_idx) if (idx % 512 == 0) { - if (YcsbConstants.kAffinitizedSession) + if (!testLoader.Options.NoThreadAffinity) session.Refresh(); session.CompletePending(false); } @@ -241,7 +243,7 @@ internal unsafe (double, double) Run(TestLoader testLoader) elapsedMs = sw.ElapsedMilliseconds; waiter.Reset(); } - double insertsPerSecond = elapsedMs == 0 ? 0 : ((double)kInitCount / elapsedMs) * 1000; + double insertsPerSecond = elapsedMs == 0 ? 0 : ((double)InitCount / elapsedMs) * 1000; Console.WriteLine(TestStats.GetLoadingTimeLine(insertsPerSecond, elapsedMs)); Console.WriteLine(TestStats.GetAddressesLine(AddressLineNum.Before, store.Log.BeginAddress, store.Log.HeadAddress, store.Log.ReadOnlyAddress, store.Log.TailAddress)); @@ -253,11 +255,11 @@ internal unsafe (double, double) Run(TestLoader testLoader) idx_ = 0; - if (YcsbConstants.kDumpDistribution) + if (testLoader.Options.DumpDistribution) Console.WriteLine(store.DumpDistribution()); // Ensure first fold-over checkpoint is fast - if (YcsbConstants.kPeriodicCheckpointMilliseconds > 0 && YcsbConstants.kPeriodicCheckpointType == CheckpointType.FoldOver) + if (testLoader.Options.PeriodicCheckpointMilliseconds > 0 && testLoader.Options.PeriodicCheckpointType == CheckpointType.FoldOver) store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, true); Console.WriteLine("Executing experiment."); @@ -278,7 +280,7 @@ internal unsafe (double, double) Run(TestLoader testLoader) Stopwatch swatch = new Stopwatch(); swatch.Start(); - if (YcsbConstants.kPeriodicCheckpointMilliseconds <= 0) + if (testLoader.Options.PeriodicCheckpointMilliseconds <= 0) { Thread.Sleep(TimeSpan.FromSeconds(testLoader.Options.RunSeconds)); } @@ -287,10 +289,10 @@ internal unsafe (double, double) Run(TestLoader testLoader) var checkpointTaken = 0; while (swatch.ElapsedMilliseconds < 1000 * testLoader.Options.RunSeconds) { - if (checkpointTaken < swatch.ElapsedMilliseconds / YcsbConstants.kPeriodicCheckpointMilliseconds) + if (checkpointTaken < swatch.ElapsedMilliseconds / testLoader.Options.PeriodicCheckpointMilliseconds) { long start = swatch.ElapsedTicks; - if (store.TakeHybridLogCheckpoint(out _, YcsbConstants.kPeriodicCheckpointType, YcsbConstants.kPeriodicCheckpointTryIncremental)) + if (store.TakeHybridLogCheckpoint(out _, testLoader.Options.PeriodicCheckpointType, testLoader.Options.PeriodicCheckpointTryIncremental)) { store.CompleteCheckpointAsync().GetAwaiter().GetResult(); var timeTaken = (swatch.ElapsedTicks - start) / TimeSpan.TicksPerMillisecond; @@ -334,7 +336,7 @@ private void SetupYcsb(int thread_idx) waiter.Wait(); - var session = store.For(functions).NewSession(null, YcsbConstants.kAffinitizedSession); + var session = store.For(functions).NewSession(null, !testLoader.Options.NoThreadAffinity); #if DASHBOARD var tstart = Stopwatch.GetTimestamp(); @@ -346,7 +348,7 @@ private void SetupYcsb(int thread_idx) Value value = default; for (long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; - chunk_idx < kInitCount; + chunk_idx < InitCount; chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize) { for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize; ++idx) @@ -449,10 +451,13 @@ void DoContinuousMeasurements() #region Load Data - internal static void CreateKeyVectors(out Key[] i_keys, out Key[] t_keys) + internal static void CreateKeyVectors(TestLoader testLoader, out Key[] i_keys, out Key[] t_keys) { - i_keys = new Key[kInitCount]; - t_keys = new Key[kTxnCount]; + InitCount = YcsbConstants.kChunkSize * (testLoader.InitCount / YcsbConstants.kChunkSize); + TxnCount = YcsbConstants.kChunkSize * (testLoader.TxnCount / YcsbConstants.kChunkSize); + + i_keys = new Key[InitCount]; + t_keys = new Key[TxnCount]; } internal class KeySetter : IKeySetter diff --git a/cs/benchmark/Options.cs b/cs/benchmark/Options.cs index b1d1ace5a..c138ed222 100644 --- a/cs/benchmark/Options.cs +++ b/cs/benchmark/Options.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using CommandLine; +using FASTER.core; namespace FASTER.benchmark { @@ -57,15 +58,46 @@ class Options HelpText = "Use synthetic data")] public bool UseSyntheticData { get; set; } - [Option("runsec", Required = false, Default = YcsbConstants.kRunSeconds, + [Option("runsec", Required = false, Default = 30, HelpText = "Number of seconds to execute experiment")] public int RunSeconds { get; set; } + [Option("sd", Required = false, Default = false, + HelpText = "Use SmallData in experiment")] + public bool UseSmallData { get; set; } + + [Option("sm", Required = false, Default = false, + HelpText = "Use Small Memory log in experiment")] + public bool UseSmallMemoryLog { get; set; } + + [Option("noaff", Required = false, Default = false, + HelpText = "Do not use thread affinitization in experiment")] + public bool NoThreadAffinity { get; set; } + + [Option("chkptms", Required = false, Default = 0, + HelpText = "If > 0, the number of milliseconds between checkpoints in experiment (else checkpointing is not done")] + public int PeriodicCheckpointMilliseconds { get; set; } + + [Option("chkptsnap", Required = false, Default = false, + HelpText = "Use Snapshot checkpoint if doing periodic checkpoints (default is FoldOver")] + public bool PeriodicCheckpointUseSnapshot { get; set; } + + [Option("chkptincr", Required = false, Default = false, + HelpText = "Try incremental checkpoint if doing periodic checkpoints (default is false")] + public bool PeriodicCheckpointTryIncremental { get; set; } + + [Option("dumpdist", Required = false, Default = false, + HelpText = "Dump the distribution of each non-empty bucket in the hash table")] + public bool DumpDistribution { get; set; } + + internal CheckpointType PeriodicCheckpointType => this.PeriodicCheckpointUseSnapshot ? CheckpointType.Snapshot : CheckpointType.FoldOver; + public string GetOptionsString() { static string boolStr(bool value) => value ? "y" : "n"; return $"d: {DistributionName.ToLower()}; n: {NumaStyle}; r: {ReadPercent}; t: {ThreadCount}; z: {LockImpl}; i: {IterationCount};" - + $" sd: {boolStr(YcsbConstants.kUseSmallData)}; sm: {boolStr(YcsbConstants.kSmallMemoryLog)}; sy: {boolStr(this.UseSyntheticData)}"; + + $" sd: {boolStr(UseSmallData)}; sm: {boolStr(UseSmallMemoryLog)}; sy: {boolStr(this.UseSyntheticData)}; noaff: {boolStr(this.NoThreadAffinity)};" + + $" chkptms: {this.PeriodicCheckpointMilliseconds}; chkpttype: {(this.PeriodicCheckpointMilliseconds > 0 ? this.PeriodicCheckpointType.ToString() : "None")}; chkptincr: {boolStr(this.PeriodicCheckpointTryIncremental)}"; } } } diff --git a/cs/benchmark/TestLoader.cs b/cs/benchmark/TestLoader.cs index 111831149..6f9057b17 100644 --- a/cs/benchmark/TestLoader.cs +++ b/cs/benchmark/TestLoader.cs @@ -32,6 +32,10 @@ class TestLoader internal KeySpanByte[] init_span_keys = default; internal KeySpanByte[] txn_span_keys = default; + internal long InitCount; + internal long TxnCount; + internal int MaxKey; + internal bool Parse(string[] args) { ParserResult result = Parser.Default.ParseArguments(args); @@ -72,6 +76,10 @@ static bool verifyOption(bool isValid, string name) if (!verifyOption(this.Options.RunSeconds >= 0, "RunSeconds")) return false; + this.InitCount = this.Options.UseSmallData ? 2500480 : 250000000; + this.TxnCount = this.Options.UseSmallData ? 10000000 : 1000000000; + this.MaxKey = this.Options.UseSmallData ? 1 << 22 : 1 << 28; + return true; } @@ -89,15 +97,15 @@ private void LoadDataThreadProc() switch (this.BenchmarkType) { case BenchmarkType.Ycsb: - FASTER_YcsbBenchmark.CreateKeyVectors(out this.init_keys, out this.txn_keys); + FASTER_YcsbBenchmark.CreateKeyVectors(this, out this.init_keys, out this.txn_keys); LoadData(this, this.init_keys, this.txn_keys, new FASTER_YcsbBenchmark.KeySetter()); break; case BenchmarkType.SpanByte: - FasterSpanByteYcsbBenchmark.CreateKeyVectors(out this.init_span_keys, out this.txn_span_keys); + FasterSpanByteYcsbBenchmark.CreateKeyVectors(this, out this.init_span_keys, out this.txn_span_keys); LoadData(this, this.init_span_keys, this.txn_span_keys, new FasterSpanByteYcsbBenchmark.KeySetter()); break; case BenchmarkType.ConcurrentDictionaryYcsb: - ConcurrentDictionary_YcsbBenchmark.CreateKeyVectors(out this.init_keys, out this.txn_keys); + ConcurrentDictionary_YcsbBenchmark.CreateKeyVectors(this, out this.init_keys, out this.txn_keys); LoadData(this, this.init_keys, this.txn_keys, new ConcurrentDictionary_YcsbBenchmark.KeySetter()); break; default: @@ -105,7 +113,7 @@ private void LoadDataThreadProc() } } - private static void LoadData(TestLoader testLoader, TKey[] init_keys, TKey[] txn_keys, TKeySetter keySetter) + private void LoadData(TestLoader testLoader, TKey[] init_keys, TKey[] txn_keys, TKeySetter keySetter) where TKeySetter : IKeySetter { if (testLoader.Options.UseSyntheticData) @@ -136,16 +144,15 @@ private static void LoadData(TestLoader testLoader, TKey[] ini } } - private static unsafe void LoadDataFromFile(string filePath, string distribution, TKey[] init_keys, TKey[] txn_keys, TKeySetter keySetter) + private unsafe void LoadDataFromFile(string filePath, string distribution, TKey[] init_keys, TKey[] txn_keys, TKeySetter keySetter) where TKeySetter : IKeySetter { string init_filename = filePath + "/load_" + distribution + "_250M_raw.dat"; string txn_filename = filePath + "/run_" + distribution + "_250M_1000M_raw.dat"; - Console.WriteLine($"loading all keys from {init_filename} into memory..."); var sw = Stopwatch.StartNew(); - if (YcsbConstants.kUseSmallData) + if (this.Options.UseSmallData) { Console.WriteLine($"loading subset of keys and txns from {txn_filename} into memory..."); using FileStream stream = File.Open(txn_filename, FileMode.Open, FileAccess.Read, FileShare.Read); @@ -153,7 +160,7 @@ private static unsafe void LoadDataFromFile(string filePath, s GCHandle chunk_handle = GCHandle.Alloc(chunk, GCHandleType.Pinned); byte* chunk_ptr = (byte*)chunk_handle.AddrOfPinnedObject(); - var initValueSet = new HashSet(); + var initValueSet = new HashSet(init_keys.Length); long init_count = 0; long txn_count = 0; @@ -208,6 +215,7 @@ private static unsafe void LoadDataFromFile(string filePath, s return; } + Console.WriteLine($"loading all keys from {init_filename} into memory..."); long count = 0; using (FileStream stream = File.Open(init_filename, FileMode.Open, FileAccess.Read, @@ -323,14 +331,14 @@ private static void LoadSyntheticData(string distribution, uin internal static string DevicePath => $"{DataPath}/hlog"; - internal string BackupPath => $"{DataPath}/{this.Distribution}_{(this.Options.UseSyntheticData ? "synthetic" : "ycsb")}_{(YcsbConstants.kUseSmallData ? "2.5M_10M" : "250M_1000M")}"; + internal string BackupPath => $"{DataPath}/{this.Distribution}_{(this.Options.UseSyntheticData ? "synthetic" : "ycsb")}_{(this.Options.UseSmallData ? "2.5M_10M" : "250M_1000M")}"; internal bool MaybeRecoverStore(FasterKV store) { // Recover database for fast benchmark repeat runs. - if (this.Options.BackupAndRestore && YcsbConstants.kPeriodicCheckpointMilliseconds <= 0) + if (this.Options.BackupAndRestore && this.Options.PeriodicCheckpointMilliseconds <= 0) { - if (YcsbConstants.kUseSmallData) + if (this.Options.UseSmallData) { Console.WriteLine("Skipping Recover() for kSmallData"); return false; @@ -357,7 +365,7 @@ internal bool MaybeRecoverStore(FasterKV store) internal void MaybeCheckpointStore(FasterKV store) { // Checkpoint database for fast benchmark repeat runs. - if (this.Options.BackupAndRestore && YcsbConstants.kPeriodicCheckpointMilliseconds <= 0) + if (this.Options.BackupAndRestore && this.Options.PeriodicCheckpointMilliseconds <= 0) { Console.WriteLine($"Checkpointing FasterKV to {this.BackupPath} for fast restart"); Stopwatch sw = Stopwatch.StartNew(); diff --git a/cs/benchmark/YcsbConstants.cs b/cs/benchmark/YcsbConstants.cs index 70a95ec9f..cbbd5e522 100644 --- a/cs/benchmark/YcsbConstants.cs +++ b/cs/benchmark/YcsbConstants.cs @@ -60,31 +60,11 @@ public static class YcsbConstants internal const string InsPerSec = "ins/sec"; internal const string OpsPerSec = "ops/sec"; -#if DEBUG - internal const bool kUseSmallData = true; - internal const bool kSmallMemoryLog = false; - internal const int kRunSeconds = 30; - internal const bool kDumpDistribution = false; - internal const bool kAffinitizedSession = true; - internal const int kPeriodicCheckpointMilliseconds = 0; -#else - internal const bool kUseSmallData = false; - internal const bool kSmallMemoryLog = false; - internal const int kRunSeconds = 30; - internal const bool kDumpDistribution = false; - internal const bool kAffinitizedSession = true; - internal const int kPeriodicCheckpointMilliseconds = 0; -#endif - internal const CheckpointType kPeriodicCheckpointType = CheckpointType.FoldOver; internal const bool kPeriodicCheckpointTryIncremental = false; internal const double SyntheticZipfTheta = 0.99; - internal const long kInitCount = kUseSmallData ? 2500480 : 250000000; - internal const long kTxnCount = kUseSmallData ? 10000000 : 1000000000; - internal const int kMaxKey = kUseSmallData ? 1 << 22 : 1 << 28; - internal const int kFileChunkSize = 4096; internal const long kChunkSize = 640; } diff --git a/cs/benchmark/scripts/compare_runs.ps1 b/cs/benchmark/scripts/compare_runs.ps1 index 3a7dc6305..9547312e8 100644 --- a/cs/benchmark/scripts/compare_runs.ps1 +++ b/cs/benchmark/scripts/compare_runs.ps1 @@ -61,6 +61,10 @@ class Result : System.IComparable, System.IEquatable[Object] { [bool]$SmallData [bool]$SmallMemory [bool]$SyntheticData + [bool]$NoAff + [int]$ChkptMs + [string]$ChkptType + [bool]$ChkptIncr Result([string]$line) { $fields = $line.Split(';') @@ -81,6 +85,10 @@ class Result : System.IComparable, System.IEquatable[Object] { "sd" { $this.SmallData = $value -eq "y" } "sm" { $this.SmallMemory = $value -eq "y" } "sy" { $this.SyntheticData = $value -eq "y" } + "noaff" { $this.NoAff = $value -eq "y" } + "chkptms" { $this.ChkptMs = $value } + "chkpttype" { $this.ChkptType = $value } + "chkptincr" { $this.ChkptIncr = $value -eq "y" } } } } @@ -95,6 +103,10 @@ class Result : System.IComparable, System.IEquatable[Object] { $this.SmallData = $other.SmallData $this.SmallMemory = $other.SmallMemory $this.SyntheticData = $other.SyntheticData + $this.NoAff = $other.NoAff + $this.ChkptMs = $other.ChkptMs + $this.ChkptType = $other.ChkptType + $this.ChkptIncr = $other.ChkptIncr } [Result] CalculateDifference([Result]$newResult) { @@ -158,11 +170,16 @@ class Result : System.IComparable, System.IEquatable[Object] { -and $this.SmallData -eq $other.SmallData -and $this.SmallMemory -eq $other.SmallMemory -and $this.SyntheticData -eq $other.SyntheticData + -and $this.NoAff -eq $other.NoAff + -and $this.ChkptMs -eq $other.ChkptMs + -and $this.ChkptType -eq $other.ChkptType + -and $this.ChkptIncr -eq $other.ChkptIncr } [int] GetHashCode() { return ($this.Numa, $this.Distribution, $this.ReadPercent, $this.ThreadCount, $this.LockMode, - $this.Iterations, $this.SmallData, $this.SmallMemory, $this.SyntheticData).GetHashCode(); + $this.Iterations, $this.SmallData, $this.SmallMemory, $this.SyntheticData, + $this.NoAff, $this.ChkptMs, $this.ChkptType, $this.ChkptIncr).GetHashCode(); } } @@ -266,7 +283,11 @@ function RenameProperties([System.Object[]]$results) { Iterations, SmallData, SmallMemory, - SyntheticData + SyntheticData, + NoAff, + ChkptMs, + ChkptType, + ChkptIncr } RenameProperties $LoadResults | Out-GridView -Title "Loading Comparison (Inserts Per Second): $OldDir -vs- $NewDir"