From 73fe128475282c7f1f7a69fdd9b488ed26e00531 Mon Sep 17 00:00:00 2001 From: Ted Hart <15467143+TedHartMS@users.noreply.github.com> Date: Wed, 31 May 2023 12:18:26 -0700 Subject: [PATCH] add Delete to YCSB Benchmark; change "-r read%" to "-rumd read% upsert% rmw% delete%" (#839) --- cs/benchmark/ConcurrentDictionaryBenchmark.cs | 63 ++++------ cs/benchmark/FasterSpanByteYcsbBenchmark.cs | 116 ++++++++---------- cs/benchmark/FasterYcsbBenchmark.cs | 114 +++++++---------- cs/benchmark/Options.cs | 15 +-- cs/benchmark/TestLoader.cs | 25 ++-- cs/benchmark/YcsbConstants.cs | 7 -- 6 files changed, 145 insertions(+), 195 deletions(-) diff --git a/cs/benchmark/ConcurrentDictionaryBenchmark.cs b/cs/benchmark/ConcurrentDictionaryBenchmark.cs index 911315933..b0f49f043 100644 --- a/cs/benchmark/ConcurrentDictionaryBenchmark.cs +++ b/cs/benchmark/ConcurrentDictionaryBenchmark.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -#pragma warning disable CS0162 // Unreachable code detected -- when switching on YcsbConstants - //#define DASHBOARD using FASTER.core; @@ -30,7 +28,7 @@ internal unsafe class ConcurrentDictionary_YcsbBenchmark readonly TestLoader testLoader; readonly int numaStyle; readonly string distribution; - readonly int readPercent; + readonly int readPercent, upsertPercent, rmwPercent; readonly Input[] input_; readonly Key[] init_keys_; @@ -50,7 +48,9 @@ internal ConcurrentDictionary_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLo txn_keys_ = t_keys_; numaStyle = testLoader.Options.NumaStyle; distribution = testLoader.Distribution; - readPercent = testLoader.Options.ReadPercent; + readPercent = testLoader.ReadPercent; + upsertPercent = testLoader.UpsertPercent; + rmwPercent = testLoader.RmwPercent; #if DASHBOARD statsWritten = new AutoResetEvent[threadCount]; @@ -93,6 +93,7 @@ private void RunYcsb(int thread_idx) Value value = default; long reads_done = 0; long writes_done = 0; + long deletes_done = 0; #if DASHBOARD var tstart = Stopwatch.GetTimestamp(); @@ -113,40 +114,27 @@ private void RunYcsb(int thread_idx) for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize && !done; ++idx) { - Op op; - int r = (int)rng.Generate(100); + int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 if (r < readPercent) - op = Op.Read; - else if (readPercent >= 0) - op = Op.Upsert; - else - op = Op.ReadModifyWrite; - - switch (op) { - case Op.Upsert: - { - store[txn_keys_[idx]] = value; - ++writes_done; - break; - } - case Op.Read: - { - if (store.TryGetValue(txn_keys_[idx], out value)) - { - ++reads_done; - } - break; - } - case Op.ReadModifyWrite: - { - store.AddOrUpdate(txn_keys_[idx], *(Value*)(input_ptr + (idx & 0x7)), (k, v) => new Value { value = v.value + (input_ptr + (idx & 0x7))->value }); - ++writes_done; - break; - } - default: - throw new InvalidOperationException("Unexpected op: " + op); + if (store.TryGetValue(txn_keys_[idx], out value)) + ++reads_done; + continue; + } + if (r < upsertPercent) + { + store[txn_keys_[idx]] = value; + ++writes_done; + continue; + } + if (r < rmwPercent) + { + store.AddOrUpdate(txn_keys_[idx], *(Value*)(input_ptr + (idx & 0x7)), (k, v) => new Value { value = v.value + (input_ptr + (idx & 0x7))->value }); + ++writes_done; + continue; } + store.Remove(txn_keys_[idx], out _); + ++deletes_done; } #if DASHBOARD @@ -168,9 +156,8 @@ private void RunYcsb(int thread_idx) sw.Stop(); - Console.WriteLine("Thread " + thread_idx + " done; " + reads_done + " reads, " + - writes_done + " writes, in " + sw.ElapsedMilliseconds + " ms."); - Interlocked.Add(ref total_ops_done, reads_done + writes_done); + Console.WriteLine($"Thread {thread_idx} done; {reads_done} reads, {writes_done} writes, {deletes_done} deletes in {sw.ElapsedMilliseconds} ms."); + Interlocked.Add(ref total_ops_done, reads_done + writes_done + deletes_done); } internal unsafe (double, double) Run(TestLoader testLoader) diff --git a/cs/benchmark/FasterSpanByteYcsbBenchmark.cs b/cs/benchmark/FasterSpanByteYcsbBenchmark.cs index adb467b73..6287becd3 100644 --- a/cs/benchmark/FasterSpanByteYcsbBenchmark.cs +++ b/cs/benchmark/FasterSpanByteYcsbBenchmark.cs @@ -21,7 +21,7 @@ internal class FasterSpanByteYcsbBenchmark readonly TestLoader testLoader; readonly ManualResetEventSlim waiter = new(); readonly int numaStyle; - readonly int readPercent; + readonly int readPercent, upsertPercent, rmwPercent; readonly FunctionsSB functions; readonly Input[] input_; @@ -52,7 +52,9 @@ internal FasterSpanByteYcsbBenchmark(KeySpanByte[] i_keys_, KeySpanByte[] t_keys init_keys_ = i_keys_; txn_keys_ = t_keys_; numaStyle = testLoader.Options.NumaStyle; - readPercent = testLoader.Options.ReadPercent; + readPercent = testLoader.ReadPercent; + upsertPercent = testLoader.UpsertPercent; + rmwPercent = testLoader.RmwPercent; functions = new FunctionsSB(); #if DASHBOARD @@ -116,6 +118,7 @@ private void RunYcsbUnsafeContext(int thread_idx) long reads_done = 0; long writes_done = 0; + long deletes_done = 0; #if DASHBOARD var tstart = Stopwatch.GetTimestamp(); @@ -142,44 +145,33 @@ private void RunYcsbUnsafeContext(int thread_idx) for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize && !done; ++idx) { - Op op; - int r = (int)rng.Generate(100); - if (r < readPercent) - op = Op.Read; - else if (readPercent >= 0) - op = Op.Upsert; - else - op = Op.ReadModifyWrite; - if (idx % 512 == 0) { uContext.Refresh(); uContext.CompletePending(false); } - switch (op) + int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 + if (r < readPercent) { - case Op.Upsert: - { - uContext.Upsert(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _value, Empty.Default, 1); - ++writes_done; - break; - } - case Op.Read: - { - uContext.Read(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _input, ref _output, Empty.Default, 1); - ++reads_done; - break; - } - case Op.ReadModifyWrite: - { - uContext.RMW(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _input, Empty.Default, 1); - ++writes_done; - break; - } - default: - throw new InvalidOperationException("Unexpected op: " + op); + uContext.Read(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _input, ref _output, Empty.Default, 1); + ++reads_done; + continue; } + if (r < upsertPercent) + { + uContext.Upsert(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _value, Empty.Default, 1); + ++writes_done; + continue; + } + if (r < rmwPercent) + { + uContext.RMW(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _input, Empty.Default, 1); + ++writes_done; + continue; + } + uContext.Delete(ref SpanByte.Reinterpret(ref txn_keys_[idx]), Empty.Default, 1); + ++deletes_done; } #if DASHBOARD @@ -214,9 +206,8 @@ private void RunYcsbUnsafeContext(int thread_idx) statsWritten[thread_idx].Set(); #endif - Console.WriteLine("Thread " + thread_idx + " done; " + reads_done + " reads, " + - writes_done + " writes, in " + sw.ElapsedMilliseconds + " ms."); - Interlocked.Add(ref total_ops_done, reads_done + writes_done); + Console.WriteLine($"Thread {thread_idx} done; {reads_done} reads, {writes_done} writes, {deletes_done} deletes in {sw.ElapsedMilliseconds} ms."); + Interlocked.Add(ref total_ops_done, reads_done + writes_done + deletes_done); } private void RunYcsbSafeContext(int thread_idx) @@ -244,6 +235,7 @@ private void RunYcsbSafeContext(int thread_idx) long reads_done = 0; long writes_done = 0; + long deletes_done = 0; #if DASHBOARD var tstart = Stopwatch.GetTimestamp(); @@ -266,15 +258,6 @@ private void RunYcsbSafeContext(int thread_idx) for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize && !done; ++idx) { - Op op; - int r = (int)rng.Generate(100); - if (r < readPercent) - op = Op.Read; - else if (readPercent >= 0) - op = Op.Upsert; - else - op = Op.ReadModifyWrite; - if (idx % 512 == 0) { if (!testLoader.Options.UseSafeContext) @@ -282,29 +265,27 @@ private void RunYcsbSafeContext(int thread_idx) session.CompletePending(false); } - switch (op) + int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 + if (r < readPercent) { - case Op.Upsert: - { - session.Upsert(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _value, Empty.Default, 1); - ++writes_done; - break; - } - case Op.Read: - { - session.Read(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _input, ref _output, Empty.Default, 1); - ++reads_done; - break; - } - case Op.ReadModifyWrite: - { - session.RMW(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _input, Empty.Default, 1); - ++writes_done; - break; - } - default: - throw new InvalidOperationException("Unexpected op: " + op); + session.Read(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _input, ref _output, Empty.Default, 1); + ++reads_done; + continue; + } + if (r < upsertPercent) + { + session.Upsert(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _value, Empty.Default, 1); + ++writes_done; + continue; + } + if (r < rmwPercent) + { + session.RMW(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _input, Empty.Default, 1); + ++writes_done; + continue; } + session.Delete(ref SpanByte.Reinterpret(ref txn_keys_[idx]), Empty.Default, 1); + ++deletes_done; } #if DASHBOARD @@ -333,9 +314,8 @@ private void RunYcsbSafeContext(int thread_idx) statsWritten[thread_idx].Set(); #endif - Console.WriteLine("Thread " + thread_idx + " done; " + reads_done + " reads, " + - writes_done + " writes, in " + sw.ElapsedMilliseconds + " ms."); - Interlocked.Add(ref total_ops_done, reads_done + writes_done); + Console.WriteLine($"Thread {thread_idx} done; {reads_done} reads, {writes_done} writes, {deletes_done} deletes in {sw.ElapsedMilliseconds} ms."); + Interlocked.Add(ref total_ops_done, reads_done + writes_done + deletes_done); } internal unsafe (double, double) Run(TestLoader testLoader) diff --git a/cs/benchmark/FasterYcsbBenchmark.cs b/cs/benchmark/FasterYcsbBenchmark.cs index 5b2365dbc..4fa679087 100644 --- a/cs/benchmark/FasterYcsbBenchmark.cs +++ b/cs/benchmark/FasterYcsbBenchmark.cs @@ -21,7 +21,7 @@ internal class FASTER_YcsbBenchmark readonly TestLoader testLoader; readonly ManualResetEventSlim waiter = new(); readonly int numaStyle; - readonly int readPercent; + readonly int readPercent, upsertPercent, rmwPercent; readonly Functions functions; readonly Input[] input_; @@ -49,7 +49,9 @@ internal FASTER_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLoader testLoade init_keys_ = i_keys_; txn_keys_ = t_keys_; numaStyle = testLoader.Options.NumaStyle; - readPercent = testLoader.Options.ReadPercent; + readPercent = testLoader.ReadPercent; + upsertPercent = testLoader.UpsertPercent; + rmwPercent = testLoader.RmwPercent; functions = new Functions(); #if DASHBOARD @@ -112,6 +114,7 @@ private void RunYcsbUnsafeContext(int thread_idx) long reads_done = 0; long writes_done = 0; + long deletes_done = 0; #if DASHBOARD var tstart = Stopwatch.GetTimestamp(); @@ -138,44 +141,33 @@ private void RunYcsbUnsafeContext(int thread_idx) for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize && !done; ++idx) { - Op op; - int r = (int)rng.Generate(100); - if (r < readPercent) - op = Op.Read; - else if (readPercent >= 0) - op = Op.Upsert; - else - op = Op.ReadModifyWrite; - if (idx % 512 == 0) { uContext.Refresh(); uContext.CompletePending(false); } - switch (op) + int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 + if (r < readPercent) { - case Op.Upsert: - { - uContext.Upsert(ref txn_keys_[idx], ref value, Empty.Default, 1); - ++writes_done; - break; - } - case Op.Read: - { - uContext.Read(ref txn_keys_[idx], ref input, ref output, Empty.Default, 1); - ++reads_done; - break; - } - case Op.ReadModifyWrite: - { - uContext.RMW(ref txn_keys_[idx], ref input_[idx & 0x7], Empty.Default, 1); - ++writes_done; - break; - } - default: - throw new InvalidOperationException("Unexpected op: " + op); + uContext.Read(ref txn_keys_[idx], ref input, ref output, Empty.Default, 1); + ++reads_done; + continue; } + if (r < upsertPercent) + { + uContext.Upsert(ref txn_keys_[idx], ref value, Empty.Default, 1); + ++writes_done; + continue; + } + if (r < rmwPercent) + { + uContext.RMW(ref txn_keys_[idx], ref input_[idx & 0x7], Empty.Default, 1); + ++writes_done; + continue; + } + uContext.Delete(ref txn_keys_[idx], Empty.Default, 1); + ++deletes_done; } #if DASHBOARD @@ -210,9 +202,8 @@ private void RunYcsbUnsafeContext(int thread_idx) statsWritten[thread_idx].Set(); #endif - Console.WriteLine("Thread " + thread_idx + " done; " + reads_done + " reads, " + - writes_done + " writes, in " + sw.ElapsedMilliseconds + " ms."); - Interlocked.Add(ref total_ops_done, reads_done + writes_done); + Console.WriteLine($"Thread {thread_idx} done; {reads_done} reads, {writes_done} writes, {deletes_done} deletes in {sw.ElapsedMilliseconds} ms."); + Interlocked.Add(ref total_ops_done, reads_done + writes_done + deletes_done); } private void RunYcsbSafeContext(int thread_idx) @@ -236,6 +227,7 @@ private void RunYcsbSafeContext(int thread_idx) long reads_done = 0; long writes_done = 0; + long deletes_done = 0; var session = store.For(functions).NewSession(); @@ -251,15 +243,6 @@ private void RunYcsbSafeContext(int thread_idx) for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize && !done; ++idx) { - Op op; - int r = (int)rng.Generate(100); - if (r < readPercent) - op = Op.Read; - else if (readPercent >= 0) - op = Op.Upsert; - else - op = Op.ReadModifyWrite; - if (idx % 512 == 0) { if (!testLoader.Options.UseSafeContext) @@ -267,29 +250,27 @@ private void RunYcsbSafeContext(int thread_idx) session.CompletePending(false); } - switch (op) + int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 + if (r < readPercent) { - case Op.Upsert: - { - session.Upsert(ref txn_keys_[idx], ref value, Empty.Default, 1); - ++writes_done; - break; - } - case Op.Read: - { - session.Read(ref txn_keys_[idx], ref input, ref output, Empty.Default, 1); - ++reads_done; - break; - } - case Op.ReadModifyWrite: - { - session.RMW(ref txn_keys_[idx], ref input_[idx & 0x7], Empty.Default, 1); - ++writes_done; - break; - } - default: - throw new InvalidOperationException("Unexpected op: " + op); + session.Read(ref txn_keys_[idx], ref input, ref output, Empty.Default, 1); + ++reads_done; + continue; + } + if (r < upsertPercent) + { + session.Upsert(ref txn_keys_[idx], ref value, Empty.Default, 1); + ++writes_done; + continue; + } + if (r < rmwPercent) + { + session.RMW(ref txn_keys_[idx], ref input_[idx & 0x7], Empty.Default, 1); + ++writes_done; + continue; } + session.Delete(ref txn_keys_[idx], Empty.Default, 1); + ++deletes_done; } } @@ -298,8 +279,7 @@ private void RunYcsbSafeContext(int thread_idx) sw.Stop(); - Console.WriteLine("Thread " + thread_idx + " done; " + reads_done + " reads, " + - writes_done + " writes, in " + sw.ElapsedMilliseconds + " ms."); + Console.WriteLine($"Thread {thread_idx} done; {reads_done} reads, {writes_done} writes, {deletes_done} deletes in {sw.ElapsedMilliseconds} ms."); Interlocked.Add(ref total_ops_done, reads_done + writes_done); } diff --git a/cs/benchmark/Options.cs b/cs/benchmark/Options.cs index 02e35c00e..013edd6f0 100644 --- a/cs/benchmark/Options.cs +++ b/cs/benchmark/Options.cs @@ -3,6 +3,7 @@ using CommandLine; using FASTER.core; +using System.Collections.Generic; namespace FASTER.benchmark { @@ -42,10 +43,6 @@ class Options HelpText = "Number of iterations of the test to run")] public int IterationCount { get; set; } - [Option('r', "read_percent", Required = false, Default = 50, - HelpText = "Percentage of reads (-1 for 100% read-modify-write")] - public int ReadPercent { get; set; } - [Option('d', "distribution", Required = false, Default = YcsbConstants.UniformDist, HelpText = "Distribution of keys in workload")] public string DistributionName { get; set; } @@ -54,6 +51,10 @@ class Options HelpText = "Seed for synthetic data distribution")] public int RandomSeed { get; set; } + [Option("rumd", Separator = ',', Required = false, Default = new[] {50,50,0,0}, + HelpText = "#,#,#,#: Percentages of [(r)eads,(u)pserts,r(m)ws,(d)eletes] (summing to 100) operations in this run")] + public IEnumerable RumdPercents { get; set; } + [Option("synth", Required = false, Default = false, HelpText = "Use synthetic data")] public bool UseSyntheticData { get; set; } @@ -83,11 +84,11 @@ class Options public int PeriodicCheckpointMilliseconds { get; set; } [Option("chkptsnap", Required = false, Default = false, - HelpText = "Use Snapshot checkpoint if doing periodic checkpoints (default is FoldOver")] + HelpText = "Use Snapshot checkpoint if doing periodic checkpoints (default is FoldOver)")] public bool PeriodicCheckpointUseSnapshot { get; set; } [Option("chkptincr", Required = false, Default = false, - HelpText = "Try incremental checkpoint if doing periodic checkpoints (default is false")] + HelpText = "Try incremental checkpoint if doing periodic checkpoints")] public bool PeriodicCheckpointTryIncremental { get; set; } [Option("dumpdist", Required = false, Default = false, @@ -99,7 +100,7 @@ class Options public string GetOptionsString() { static string boolStr(bool value) => value ? "y" : "n"; - return $"d: {DistributionName.ToLower()}; n: {NumaStyle}; r: {ReadPercent}; t: {ThreadCount}; z: {LockingMode}; i: {IterationCount}; hp: {HashPacking}" + return $"d: {DistributionName.ToLower()}; n: {NumaStyle}; rumd: {string.Join(',', RumdPercents)}; t: {ThreadCount}; z: {LockingMode}; i: {IterationCount}; hp: {HashPacking}" + $" sd: {boolStr(UseSmallData)}; sm: {boolStr(UseSmallMemoryLog)}; sy: {boolStr(this.UseSyntheticData)}; safectx: {boolStr(this.UseSafeContext)};" + $" chkptms: {this.PeriodicCheckpointMilliseconds}; chkpttype: {(this.PeriodicCheckpointMilliseconds > 0 ? this.PeriodicCheckpointType.ToString() : "None")};" + $" chkptincr: {boolStr(this.PeriodicCheckpointTryIncremental)}"; diff --git a/cs/benchmark/TestLoader.cs b/cs/benchmark/TestLoader.cs index 9e5f73739..419d5dfa8 100644 --- a/cs/benchmark/TestLoader.cs +++ b/cs/benchmark/TestLoader.cs @@ -7,7 +7,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Runtime.CompilerServices; +using System.Linq; using System.Runtime.InteropServices; using System.Threading; @@ -35,21 +35,22 @@ class TestLoader internal readonly bool RecoverMode; internal readonly bool error; + // RUMD percentages. Delete is not a percent; it will fire automatically if the others do not sum to 100, saving an 'if'. + internal readonly int ReadPercent, UpsertPercent, RmwPercent; internal TestLoader(string[] args) { error = true; ParserResult result = Parser.Default.ParseArguments(args); if (result.Tag == ParserResultType.NotParsed) - { return; - } + Options = result.MapResult(o => o, xs => new Options()); - static bool verifyOption(bool isValid, string name) + static bool verifyOption(bool isValid, string name, string info = null) { if (!isValid) - Console.WriteLine($"Invalid {name} argument"); + Console.WriteLine($"Invalid {name} argument" + (string.IsNullOrEmpty(info) ? string.Empty : $": {info}")); return isValid; } @@ -75,9 +76,6 @@ static bool verifyOption(bool isValid, string name) if (!verifyOption(Options.HashPacking > 0, "Iteration Count")) return; - if (!verifyOption(Options.ReadPercent >= -1 && Options.ReadPercent <= 100, "Read Percent")) - return; - this.Distribution = Options.DistributionName.ToLower(); if (!verifyOption(this.Distribution == YcsbConstants.UniformDist || this.Distribution == YcsbConstants.ZipfDist, "Distribution")) return; @@ -85,6 +83,17 @@ static bool verifyOption(bool isValid, string name) if (!verifyOption(this.Options.RunSeconds >= 0, "RunSeconds")) return; + var rumdPercents = Options.RumdPercents.ToArray(); // Will be non-null because we specified a default + if (!verifyOption(rumdPercents.Length == 4 && Options.RumdPercents.Sum() == 100 && !Options.RumdPercents.Any(x => x < 0), "rmud", + "Percentages of [(r)eads,(u)pserts,r(m)ws,(d)eletes] must be empty or must sum to 100 with no negative elements")) + return; + else + { + this.ReadPercent = rumdPercents[0]; + this.UpsertPercent = this.ReadPercent + rumdPercents[1]; + this.RmwPercent = this.UpsertPercent + rumdPercents[2]; + } + this.InitCount = this.Options.UseSmallData ? 2500480 : 250000000; this.TxnCount = this.Options.UseSmallData ? 10000000 : 1000000000; this.MaxKey = this.Options.UseSmallData ? 1 << 22 : 1 << 28; diff --git a/cs/benchmark/YcsbConstants.cs b/cs/benchmark/YcsbConstants.cs index ab9a33c22..9d90c390c 100644 --- a/cs/benchmark/YcsbConstants.cs +++ b/cs/benchmark/YcsbConstants.cs @@ -36,13 +36,6 @@ enum StatsLineNum : int FinalTrimmedOps = 21 } - public enum Op : ulong - { - Upsert = 0, - Read = 1, - ReadModifyWrite = 2 - } - public static class YcsbConstants { internal const string UniformDist = "uniform"; // Uniformly random distribution of keys