Solve MinIterationTime warning

[swtrse]

Hello,

I have a Benchmark that looks like

namespace Cobra.Exoda.Matching.Benchmarks.Base;

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Jobs;
using JetBrains.Annotations;

[CategoriesColumn]
[MinColumn]
[MaxColumn]
[MemoryDiagnoser(false)]
[HideColumns("Job", "Type")]
[ShortRunJob(RuntimeMoniker.Net80)]
[ShortRunJob(RuntimeMoniker.Net90)]
//[ShortRunJob(RuntimeMoniker.NativeAot80)] // Bug
//[ShortRunJob(RuntimeMoniker.NativeAot90)] // Bug
public class BenchmarkBase
{
	[Params(0, 10, 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000, 100_000_000)]
	public int Prefill { get; [UsedImplicitly] set; }

	[Params(10)]
	public int Inserts { get; [UsedImplicitly] set; }

	[Params(1_00, 2_000, 5_000, 10_000)] public int PriceRange { get; [UsedImplicitly] set; }
}

namespace Cobra.Exoda.Matching.Benchmarks;

using System.Collections.Immutable;
using System.ComponentModel.DataAnnotations;
using System.Numerics;
using Base;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNetVisualizer;
using JetBrains.Annotations;
using Nethermind.Int256;
using SortedDictionary.BigInteger;
using SortedDictionary.UInt256;
using Utils;

[PublicAPI]
[RichHtmlExporter("Benchmark of Collection Performance",
                  ["Prefill", "Inserts", "PriceRange"],
                  ["Mean", "Allocated"],
                  ["Mean", "Min", "Max", "Allocated"],
                  dividerMode: RenderTableDividerMode.SeparateTables,
                  htmlWrapMode: HtmlDocumentWrapMode.RichDataTables)]
[GenericTypeArguments(typeof(OrderDataBigIntegerKey), typeof(OrderDataBigInteger))]
[GenericTypeArguments(typeof(OrderDataUInt256Key), typeof(OrderDataUInt256))]
[Display(Name = "Benchmark of Flat Dictionary Item Insert for IDictionary<{0}, {1}>", GroupName = "Benchmark of Dictionaries Item Insert")]

// ReSharper disable once ClassCanBeSealed.Global
public class AddItemBenchmarkFlat<TKey, TData> : BenchmarkBase
	where TKey : notnull
	where TData : notnull
{
	[GlobalSetup]
	public void A_Global_Setup()
	{
		_orderDataKeys = new TKey[Inserts];
		_orderDataArray = new TData[Inserts];
		_prefillArray = new TData[Prefill];
		var rnd = new Random();
		if (typeof(TData) == typeof(OrderDataBigInteger))
		{
			Parallel.For(0,
			             Inserts,
			             i =>
			             {
				             var price = new BigInteger(rnd.NextInt64(0, PriceRange));
				             var orderId = new Guid(i + Prefill, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
				             _orderDataKeys[i] = (dynamic)new OrderDataBigIntegerKey(price, orderId);
				             _orderDataArray[i] = (dynamic)new OrderDataBigInteger(price, orderId, 1);
			             });
			Parallel.For(0,
			             Prefill,
			             i =>
			             {
				             var price = new BigInteger(rnd.NextInt64(0, PriceRange));
				             var orderId = new Guid(i, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
				             _prefillArray[i] = (dynamic)new OrderDataBigInteger(price, orderId, 1);
			             });
		}
		else if (typeof(TData) == typeof(OrderDataUInt256))
		{
			Parallel.For(0,
			             Inserts,
			             i =>
			             {
				             var price = new UInt256((ulong)rnd.NextInt64(0, PriceRange));
				             var orderId = new Guid(i + Prefill, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
				             _orderDataKeys[i] = (dynamic)new OrderDataUInt256Key(price, orderId);
				             _orderDataArray[i] = (dynamic)new OrderDataUInt256(price, orderId, 1);
			             });
			Parallel.For(0,
			             Prefill,
			             i =>
			             {
				             var price = new UInt256((ulong)rnd.NextInt64(0, PriceRange));
				             var orderId = new Guid(i, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
				             _prefillArray[i] = (dynamic)new OrderDataUInt256(price, orderId, 1);
			             });
		}
		else { throw new Exception("Unsupported Type"); }
	}

	[IterationSetup(Target = "SortedDictionary_AddItem")]
	public void A_Iteration_Setup_SortedDictionary()
	{
		if (typeof(TData) == typeof(OrderDataBigInteger))
		{
			_collection =
				(dynamic)new SortedDictionary<OrderDataBigIntegerKey, OrderDataBigInteger>(((OrderDataBigInteger[])(dynamic)_prefillArray).ToDictionary(static pair =>
						                                                                           new OrderDataBigIntegerKey(pair.Price, pair.OrderId),
					                                                                           static pair => pair));
		}
		else
		{
			_collection =
				(dynamic)new SortedDictionary<OrderDataUInt256Key, OrderDataUInt256>(((OrderDataUInt256[])(dynamic)_prefillArray).ToDictionary(static pair =>
						                                                                     new OrderDataUInt256Key(pair.Price, pair.OrderId),
					                                                                     static pair => pair));
		}
	}

	[IterationSetup(Target = "SortedImmutableDictionary_AddItem")]
	public void A_Iteration_Setup_SortedImmutableDictionary()
	{
		if (typeof(TData) == typeof(OrderDataBigInteger))
		{
			_collection =
				(dynamic)((OrderDataBigInteger[])(dynamic)_prefillArray).ToImmutableSortedDictionary(static p => new OrderDataBigIntegerKey(p.Price, p.OrderId), static p => p);
		}
		else
		{
			_collection = (dynamic)((OrderDataUInt256[])(dynamic)_prefillArray).ToImmutableSortedDictionary(static p => new OrderDataUInt256Key(p.Price, p.OrderId), static p => p);
		}
	}

	[IterationSetup(Target = "SortedList_AddItem")]
	public void A_Iteration_Setup_SortedList()
	{
		if (typeof(TData) == typeof(OrderDataBigInteger))
		{
			_collection =
				(dynamic)new SortedList<OrderDataBigIntegerKey, OrderDataBigInteger>(((OrderDataBigInteger[])(dynamic)_prefillArray).ToDictionary(static pair =>
						                                                                     new OrderDataBigIntegerKey(pair.Price, pair.OrderId),
					                                                                     static pair => pair));
		}
		else
		{
			_collection =
				(dynamic)new SortedList<OrderDataUInt256Key, OrderDataUInt256>(((OrderDataUInt256[])(dynamic)_prefillArray).ToDictionary(static pair =>
						                                                               new OrderDataUInt256Key(pair.Price, pair.OrderId),
					                                                               static pair => pair));
		}
	}

	[Benchmark(Baseline = true)]
	[BenchmarkCategory("AddItem")]
	public void SortedDictionary_AddItem()
	{
		BenchmarkHelper.FlatCollection(_orderDataKeys, _orderDataArray, ref _collection);
	}

	[Benchmark]
	[BenchmarkCategory("AddItem")]
	public void SortedImmutableDictionary_AddItem()
	{
		var immutable = (IImmutableDictionary<TKey, TData>)_collection;
		BenchmarkHelper.FlatImmutableCollection(_orderDataKeys, _orderDataArray, ref immutable);
	}

	[Benchmark]
	[BenchmarkCategory("AddItem")]
	public void SortedList_AddItem()
	{
		BenchmarkHelper.FlatCollection(_orderDataKeys, _orderDataArray, ref _collection);
	}

	// ReSharper disable NullableWarningSuppressionIsUsed
	private IDictionary<TKey, TData> _collection = default!;

	private TData[] _prefillArray = default!;

	public string[] GetDataTypeName { get; } = [typeof(TData) == typeof(OrderDataBigInteger) ? "BigInteger" : "UInt256"];

	[ParamsSource(nameof(GetDataTypeName), Priority = -1)]
	public string DataType { get; set; } = default!;
	// ReSharper restore NullableWarningSuppressionIsUsed

	// ReSharper disable NullableWarningSuppressionIsUsed
	private TData[] _orderDataArray = default!;

	private TKey[] _orderDataKeys = default!;
	// ReSharper restore NullableWarningSuppressionIsUsed
}

What I want to compare is the performance between the different variants.
However I get the MinIterationTime warning. Since my time is far away from the needed minimum iteration time I think I need to rewrite the tests but I have absolutely no idea how to do that and achieve the same tests. What I do not want is to add the time to prepare the collection to the performance benchmark time.

[timcassell]

Add a const int OpCount = 100_000;
Run the work in a loop for (int i = 0; i < OpCount; ++i).
Attribute your benchmark like this [Benchmark(OperationsPerInvoke = OpCount)].
Adjust OpCount value until the warning goes away.
If necessary, group your setup data into an array the size of OpCount, and loop to fill it in the [IterationSetup].

[adamsitnik]

A sample benchmark that does what @timcassell has described: https://github.com/dotnet/performance/blob/main/src/benchmarks/micro/libraries/System.Collections/Sort.cs

[swtrse]

thx that's what I was looking for.
@timcassell I guess OperationsPerInvoke will calculate it down to time of one operation. Am I right?

[adamsitnik]

I guess OperationsPerInvoke will calculate it down to time of one operation. Am I right?

Yes, the results will be scaled.