Solve MinIterationTime warning
swtrse opened this issue · 4 comments
swtrse commented
Hello,
I have a Benchmark that looks like
namespace Cobra.Exoda.Matching.Benchmarks.Base;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Jobs;
using JetBrains.Annotations;
[CategoriesColumn]
[MinColumn]
[MaxColumn]
[MemoryDiagnoser(false)]
[HideColumns("Job", "Type")]
[ShortRunJob(RuntimeMoniker.Net80)]
[ShortRunJob(RuntimeMoniker.Net90)]
//[ShortRunJob(RuntimeMoniker.NativeAot80)] // Bug
//[ShortRunJob(RuntimeMoniker.NativeAot90)] // Bug
public class BenchmarkBase
{
[Params(0, 10, 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000, 100_000_000)]
public int Prefill { get; [UsedImplicitly] set; }
[Params(10)]
public int Inserts { get; [UsedImplicitly] set; }
[Params(1_00, 2_000, 5_000, 10_000)] public int PriceRange { get; [UsedImplicitly] set; }
}
namespace Cobra.Exoda.Matching.Benchmarks;
using System.Collections.Immutable;
using System.ComponentModel.DataAnnotations;
using System.Numerics;
using Base;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNetVisualizer;
using JetBrains.Annotations;
using Nethermind.Int256;
using SortedDictionary.BigInteger;
using SortedDictionary.UInt256;
using Utils;
[PublicAPI]
[RichHtmlExporter("Benchmark of Collection Performance",
["Prefill", "Inserts", "PriceRange"],
["Mean", "Allocated"],
["Mean", "Min", "Max", "Allocated"],
dividerMode: RenderTableDividerMode.SeparateTables,
htmlWrapMode: HtmlDocumentWrapMode.RichDataTables)]
[GenericTypeArguments(typeof(OrderDataBigIntegerKey), typeof(OrderDataBigInteger))]
[GenericTypeArguments(typeof(OrderDataUInt256Key), typeof(OrderDataUInt256))]
[Display(Name = "Benchmark of Flat Dictionary Item Insert for IDictionary<{0}, {1}>", GroupName = "Benchmark of Dictionaries Item Insert")]
// ReSharper disable once ClassCanBeSealed.Global
public class AddItemBenchmarkFlat<TKey, TData> : BenchmarkBase
where TKey : notnull
where TData : notnull
{
[GlobalSetup]
public void A_Global_Setup()
{
_orderDataKeys = new TKey[Inserts];
_orderDataArray = new TData[Inserts];
_prefillArray = new TData[Prefill];
var rnd = new Random();
if (typeof(TData) == typeof(OrderDataBigInteger))
{
Parallel.For(0,
Inserts,
i =>
{
var price = new BigInteger(rnd.NextInt64(0, PriceRange));
var orderId = new Guid(i + Prefill, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
_orderDataKeys[i] = (dynamic)new OrderDataBigIntegerKey(price, orderId);
_orderDataArray[i] = (dynamic)new OrderDataBigInteger(price, orderId, 1);
});
Parallel.For(0,
Prefill,
i =>
{
var price = new BigInteger(rnd.NextInt64(0, PriceRange));
var orderId = new Guid(i, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
_prefillArray[i] = (dynamic)new OrderDataBigInteger(price, orderId, 1);
});
}
else if (typeof(TData) == typeof(OrderDataUInt256))
{
Parallel.For(0,
Inserts,
i =>
{
var price = new UInt256((ulong)rnd.NextInt64(0, PriceRange));
var orderId = new Guid(i + Prefill, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
_orderDataKeys[i] = (dynamic)new OrderDataUInt256Key(price, orderId);
_orderDataArray[i] = (dynamic)new OrderDataUInt256(price, orderId, 1);
});
Parallel.For(0,
Prefill,
i =>
{
var price = new UInt256((ulong)rnd.NextInt64(0, PriceRange));
var orderId = new Guid(i, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
_prefillArray[i] = (dynamic)new OrderDataUInt256(price, orderId, 1);
});
}
else { throw new Exception("Unsupported Type"); }
}
[IterationSetup(Target = "SortedDictionary_AddItem")]
public void A_Iteration_Setup_SortedDictionary()
{
if (typeof(TData) == typeof(OrderDataBigInteger))
{
_collection =
(dynamic)new SortedDictionary<OrderDataBigIntegerKey, OrderDataBigInteger>(((OrderDataBigInteger[])(dynamic)_prefillArray).ToDictionary(static pair =>
new OrderDataBigIntegerKey(pair.Price, pair.OrderId),
static pair => pair));
}
else
{
_collection =
(dynamic)new SortedDictionary<OrderDataUInt256Key, OrderDataUInt256>(((OrderDataUInt256[])(dynamic)_prefillArray).ToDictionary(static pair =>
new OrderDataUInt256Key(pair.Price, pair.OrderId),
static pair => pair));
}
}
[IterationSetup(Target = "SortedImmutableDictionary_AddItem")]
public void A_Iteration_Setup_SortedImmutableDictionary()
{
if (typeof(TData) == typeof(OrderDataBigInteger))
{
_collection =
(dynamic)((OrderDataBigInteger[])(dynamic)_prefillArray).ToImmutableSortedDictionary(static p => new OrderDataBigIntegerKey(p.Price, p.OrderId), static p => p);
}
else
{
_collection = (dynamic)((OrderDataUInt256[])(dynamic)_prefillArray).ToImmutableSortedDictionary(static p => new OrderDataUInt256Key(p.Price, p.OrderId), static p => p);
}
}
[IterationSetup(Target = "SortedList_AddItem")]
public void A_Iteration_Setup_SortedList()
{
if (typeof(TData) == typeof(OrderDataBigInteger))
{
_collection =
(dynamic)new SortedList<OrderDataBigIntegerKey, OrderDataBigInteger>(((OrderDataBigInteger[])(dynamic)_prefillArray).ToDictionary(static pair =>
new OrderDataBigIntegerKey(pair.Price, pair.OrderId),
static pair => pair));
}
else
{
_collection =
(dynamic)new SortedList<OrderDataUInt256Key, OrderDataUInt256>(((OrderDataUInt256[])(dynamic)_prefillArray).ToDictionary(static pair =>
new OrderDataUInt256Key(pair.Price, pair.OrderId),
static pair => pair));
}
}
[Benchmark(Baseline = true)]
[BenchmarkCategory("AddItem")]
public void SortedDictionary_AddItem()
{
BenchmarkHelper.FlatCollection(_orderDataKeys, _orderDataArray, ref _collection);
}
[Benchmark]
[BenchmarkCategory("AddItem")]
public void SortedImmutableDictionary_AddItem()
{
var immutable = (IImmutableDictionary<TKey, TData>)_collection;
BenchmarkHelper.FlatImmutableCollection(_orderDataKeys, _orderDataArray, ref immutable);
}
[Benchmark]
[BenchmarkCategory("AddItem")]
public void SortedList_AddItem()
{
BenchmarkHelper.FlatCollection(_orderDataKeys, _orderDataArray, ref _collection);
}
// ReSharper disable NullableWarningSuppressionIsUsed
private IDictionary<TKey, TData> _collection = default!;
private TData[] _prefillArray = default!;
public string[] GetDataTypeName { get; } = [typeof(TData) == typeof(OrderDataBigInteger) ? "BigInteger" : "UInt256"];
[ParamsSource(nameof(GetDataTypeName), Priority = -1)]
public string DataType { get; set; } = default!;
// ReSharper restore NullableWarningSuppressionIsUsed
// ReSharper disable NullableWarningSuppressionIsUsed
private TData[] _orderDataArray = default!;
private TKey[] _orderDataKeys = default!;
// ReSharper restore NullableWarningSuppressionIsUsed
}
What I want to compare is the performance between the different variants.
However I get the MinIterationTime warning. Since my time is far away from the needed minimum iteration time I think I need to rewrite the tests but I have absolutely no idea how to do that and achieve the same tests. What I do not want is to add the time to prepare the collection to the performance benchmark time.
timcassell commented
Add a const int OpCount = 100_000;
Run the work in a loop for (int i = 0; i < OpCount; ++i).
Attribute your benchmark like this [Benchmark(OperationsPerInvoke = OpCount)].
Adjust OpCount value until the warning goes away.
If necessary, group your setup data into an array the size of OpCount, and loop to fill it in the [IterationSetup].
adamsitnik commented
A sample benchmark that does what @timcassell has described: https://github.com/dotnet/performance/blob/main/src/benchmarks/micro/libraries/System.Collections/Sort.cs
swtrse commented
thx that's what I was looking for.
@timcassell I guess OperationsPerInvoke will calculate it down to time of one operation. Am I right?
adamsitnik commented
I guess OperationsPerInvoke will calculate it down to time of one operation. Am I right?
Yes, the results will be scaled.