diff --git a/docs/articles/samples/IntroCategoryDiscoverer.md b/docs/articles/samples/IntroCategoryDiscoverer.md new file mode 100644 index 0000000000..0f268ac913 --- /dev/null +++ b/docs/articles/samples/IntroCategoryDiscoverer.md @@ -0,0 +1,26 @@ +--- +uid: BenchmarkDotNet.Samples.IntroCategoryDiscoverer +--- + +## Sample: IntroCategoryDiscoverer + +The category discovery strategy can be overridden using an instance of `ICategoryDiscoverer`. + +### Source code + +[!code-csharp[IntroCategoryDiscoverer.cs](../../../samples/BenchmarkDotNet.Samples/IntroCategoryDiscoverer.cs)] + +### Output + +```markdown +| Method | Categories | Mean | Error | +|------- |----------- |---------:|------:| +| Bar | All,B | 126.5 us | NA | +| Foo | All,F | 114.0 us | NA | +``` + +### Links + +* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroCategoryDiscoverer + +--- \ No newline at end of file diff --git a/docs/articles/samples/toc.yml b/docs/articles/samples/toc.yml index 098e457dd8..e5c84fee1d 100644 --- a/docs/articles/samples/toc.yml +++ b/docs/articles/samples/toc.yml @@ -12,6 +12,8 @@ href: IntroCategories.md - name: IntroCategoryBaseline href: IntroCategoryBaseline.md +- name: IntroCategoryDiscoverer + href: IntroCategoryDiscoverer.md - name: IntroColdStart href: IntroColdStart.md - name: IntroComparableComplexParam diff --git a/samples/BenchmarkDotNet.Samples/IntroCategoryDiscoverer.cs b/samples/BenchmarkDotNet.Samples/IntroCategoryDiscoverer.cs new file mode 100644 index 0000000000..92383eb2c8 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroCategoryDiscoverer.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Reflection; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Samples +{ + [DryJob] + [CategoriesColumn] + [CustomCategoryDiscoverer] + public class IntroCategoryDiscoverer + { + private class CustomCategoryDiscoverer : DefaultCategoryDiscoverer + { + public override string[] GetCategories(MethodInfo method) + { + var categories = new List(); + categories.AddRange(base.GetCategories(method)); + categories.Add("All"); + categories.Add(method.Name.Substring(0, 1)); + return categories.ToArray(); + } + } + + [AttributeUsage(AttributeTargets.Class)] + private class CustomCategoryDiscovererAttribute : Attribute, IConfigSource + { + public CustomCategoryDiscovererAttribute() + { + Config = ManualConfig.CreateEmpty() + .WithCategoryDiscoverer(new CustomCategoryDiscoverer()); + } + + public IConfig Config { get; } + } + + + [Benchmark] + public void Foo() { } + + [Benchmark] + public void Bar() { } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/CategoryDiscovererAttribute.cs b/src/BenchmarkDotNet/Attributes/CategoryDiscovererAttribute.cs new file mode 100644 index 0000000000..764042df28 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/CategoryDiscovererAttribute.cs @@ -0,0 +1,17 @@ +using System; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Attributes +{ + [AttributeUsage(AttributeTargets.Class)] + public class CategoryDiscovererAttribute : Attribute, IConfigSource + { + public CategoryDiscovererAttribute(bool inherit = true) + { + Config = ManualConfig.CreateEmpty().WithCategoryDiscoverer(new DefaultCategoryDiscoverer(inherit)); + } + + public IConfig Config { get; } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs index 764d798933..2cbdff2461 100644 --- a/src/BenchmarkDotNet/Configs/DebugConfig.cs +++ b/src/BenchmarkDotNet/Configs/DebugConfig.cs @@ -12,6 +12,7 @@ using BenchmarkDotNet.Order; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.InProcess.Emit; using BenchmarkDotNet.Validators; @@ -66,6 +67,7 @@ public abstract class DebugConfig : IConfig public IEnumerable GetColumnHidingRules() => Array.Empty(); public IOrderer Orderer => DefaultOrderer.Instance; + public ICategoryDiscoverer? CategoryDiscoverer => DefaultCategoryDiscoverer.Instance; public SummaryStyle SummaryStyle => SummaryStyle.Default; public ConfigUnionRule UnionRule => ConfigUnionRule.Union; public TimeSpan BuildTimeout => DefaultConfig.Instance.BuildTimeout; diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index 9a8f0345f0..8d1df6285b 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -13,6 +13,7 @@ using BenchmarkDotNet.Order; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.Configs @@ -72,6 +73,7 @@ public IEnumerable GetValidators() } public IOrderer Orderer => null; + public ICategoryDiscoverer? CategoryDiscoverer => null; public ConfigUnionRule UnionRule => ConfigUnionRule.Union; diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs index beb30a33e6..46a53692c2 100644 --- a/src/BenchmarkDotNet/Configs/IConfig.cs +++ b/src/BenchmarkDotNet/Configs/IConfig.cs @@ -10,6 +10,7 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Order; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; using JetBrains.Annotations; @@ -30,6 +31,7 @@ public interface IConfig IEnumerable GetColumnHidingRules(); IOrderer? Orderer { get; } + ICategoryDiscoverer? CategoryDiscoverer { get; } SummaryStyle SummaryStyle { get; } @@ -57,4 +59,4 @@ public interface IConfig /// IReadOnlyList ConfigAnalysisConclusion { get; } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index d4a1cfd47a..b5d84eaf56 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -14,7 +14,6 @@ using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; -using JetBrains.Annotations; using RunMode = BenchmarkDotNet.Diagnosers.RunMode; namespace BenchmarkDotNet.Configs @@ -50,6 +49,7 @@ internal ImmutableConfig( string artifactsPath, CultureInfo cultureInfo, IOrderer orderer, + ICategoryDiscoverer categoryDiscoverer, SummaryStyle summaryStyle, ConfigOptions options, TimeSpan buildTimeout, @@ -70,6 +70,7 @@ internal ImmutableConfig( ArtifactsPath = artifactsPath; CultureInfo = cultureInfo; Orderer = orderer; + CategoryDiscoverer = categoryDiscoverer; SummaryStyle = summaryStyle; Options = options; BuildTimeout = buildTimeout; @@ -81,6 +82,7 @@ internal ImmutableConfig( public CultureInfo CultureInfo { get; } public ConfigOptions Options { get; } public IOrderer Orderer { get; } + public ICategoryDiscoverer CategoryDiscoverer { get; } public SummaryStyle SummaryStyle { get; } public TimeSpan BuildTimeout { get; } diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 42c05f77ef..e85abf926d 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -7,6 +7,7 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Order; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.Configs @@ -69,6 +70,7 @@ public static ImmutableConfig Create(IConfig source) source.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath, source.CultureInfo, source.Orderer ?? DefaultOrderer.Instance, + source.CategoryDiscoverer ?? DefaultCategoryDiscoverer.Instance, source.SummaryStyle ?? SummaryStyle.Default, source.Options, source.BuildTimeout, diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index 9f632c03d0..27eca81863 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -13,6 +13,7 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Order; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; using JetBrains.Annotations; @@ -51,6 +52,7 @@ public class ManualConfig : IConfig [PublicAPI] public string ArtifactsPath { get; set; } [PublicAPI] public CultureInfo CultureInfo { get; set; } [PublicAPI] public IOrderer Orderer { get; set; } + [PublicAPI] public ICategoryDiscoverer CategoryDiscoverer { get; set; } [PublicAPI] public SummaryStyle SummaryStyle { get; set; } [PublicAPI] public TimeSpan BuildTimeout { get; set; } = DefaultConfig.Instance.BuildTimeout; @@ -92,6 +94,12 @@ public ManualConfig WithOrderer(IOrderer orderer) return this; } + public ManualConfig WithCategoryDiscoverer(ICategoryDiscoverer categoryDiscoverer) + { + CategoryDiscoverer = categoryDiscoverer; + return this; + } + public ManualConfig WithBuildTimeout(TimeSpan buildTimeout) { BuildTimeout = buildTimeout; @@ -247,6 +255,7 @@ public void Add(IConfig config) hardwareCounters.AddRange(config.GetHardwareCounters()); filters.AddRange(config.GetFilters()); Orderer = config.Orderer ?? Orderer; + CategoryDiscoverer = config.CategoryDiscoverer ?? CategoryDiscoverer; ArtifactsPath = config.ArtifactsPath ?? ArtifactsPath; CultureInfo = config.CultureInfo ?? CultureInfo; SummaryStyle = config.SummaryStyle ?? SummaryStyle; diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs index ed546e37c6..dbe350b2c8 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs @@ -51,7 +51,8 @@ private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type type, Met var iterationSetupMethods = GetAttributedMethods(allMethods, "IterationSetup"); var iterationCleanupMethods = GetAttributedMethods(allMethods, "IterationCleanup"); - var targets = GetTargets(benchmarkMethods, type, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods).ToArray(); + var targets = GetTargets(benchmarkMethods, type, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods, + configPerType).ToArray(); var parameterDefinitions = GetParameterDefinitions(type); var parameterInstancesList = parameterDefinitions.Expand(configPerType.SummaryStyle); @@ -115,7 +116,8 @@ private static IEnumerable GetTargets( Tuple[] globalSetupMethods, Tuple[] globalCleanupMethods, Tuple[] iterationSetupMethods, - Tuple[] iterationCleanupMethods) + Tuple[] iterationCleanupMethods, + IConfig config) { return targetMethods .Select(methodInfo => CreateDescriptor(type, @@ -125,7 +127,8 @@ private static IEnumerable GetTargets( GetTargetedMatchingMethod(methodInfo, iterationSetupMethods), GetTargetedMatchingMethod(methodInfo, iterationCleanupMethods), methodInfo.ResolveAttribute(), - targetMethods)); + targetMethods, + config)); } private static MethodInfo GetTargetedMatchingMethod(MethodInfo benchmarkMethod, Tuple[] methods) @@ -152,8 +155,10 @@ private static Descriptor CreateDescriptor( MethodInfo iterationSetupMethod, MethodInfo iterationCleanupMethod, BenchmarkAttribute attr, - MethodInfo[] targetMethods) + MethodInfo[] targetMethods, + IConfig config) { + var categoryDiscoverer = config.CategoryDiscoverer ?? DefaultCategoryDiscoverer.Instance; var target = new Descriptor( type, methodInfo, @@ -163,7 +168,7 @@ private static Descriptor CreateDescriptor( iterationCleanupMethod, attr.Description, baseline: attr.Baseline, - categories: GetCategories(methodInfo), + categories: categoryDiscoverer.GetCategories(methodInfo), operationsPerInvoke: attr.OperationsPerInvoke, methodIndex: Array.IndexOf(targetMethods, methodInfo)); AssertMethodHasCorrectSignature("Benchmark", methodInfo); @@ -245,21 +250,6 @@ private static IEnumerable GetArgumentsDefinitions(MethodInf yield return SmartParamBuilder.CreateForArguments(benchmark, parameterDefinitions, valuesInfo, sourceIndex, summaryStyle); } - private static string[] GetCategories(MethodInfo method) - { - var attributes = new List(); - attributes.AddRange(method.GetCustomAttributes(typeof(BenchmarkCategoryAttribute), true).OfType()); - var type = method.ReflectedType; - if (type != null) - { - attributes.AddRange(type.GetTypeInfo().GetCustomAttributes(typeof(BenchmarkCategoryAttribute), true).OfType()); - attributes.AddRange(type.GetTypeInfo().Assembly.GetCustomAttributes().OfType()); - } - if (attributes.Count == 0) - return Array.Empty(); - return attributes.SelectMany(attr => attr.Categories).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); - } - private static ImmutableArray GetFilteredBenchmarks(IEnumerable benchmarks, IEnumerable filters) => benchmarks.Where(benchmark => filters.All(filter => filter.Predicate(benchmark))).ToImmutableArray(); diff --git a/src/BenchmarkDotNet/Running/DefaultCategoryDiscoverer.cs b/src/BenchmarkDotNet/Running/DefaultCategoryDiscoverer.cs new file mode 100644 index 0000000000..f883fbe744 --- /dev/null +++ b/src/BenchmarkDotNet/Running/DefaultCategoryDiscoverer.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using BenchmarkDotNet.Attributes; + +namespace BenchmarkDotNet.Running +{ + public class DefaultCategoryDiscoverer : ICategoryDiscoverer + { + public static readonly ICategoryDiscoverer Instance = new DefaultCategoryDiscoverer(); + + private readonly bool inherit; + + public DefaultCategoryDiscoverer(bool inherit = true) + { + this.inherit = inherit; + } + + public virtual string[] GetCategories(MethodInfo method) + { + var attributes = new List(); + attributes.AddRange(method.GetCustomAttributes(typeof(BenchmarkCategoryAttribute), inherit).OfType()); + var type = method.ReflectedType; + if (type != null) + { + attributes.AddRange(type.GetTypeInfo().GetCustomAttributes(typeof(BenchmarkCategoryAttribute), inherit).OfType()); + attributes.AddRange(type.GetTypeInfo().Assembly.GetCustomAttributes().OfType()); + } + if (attributes.Count == 0) + return Array.Empty(); + return attributes.SelectMany(attr => attr.Categories).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/ICategoryDiscoverer.cs b/src/BenchmarkDotNet/Running/ICategoryDiscoverer.cs new file mode 100644 index 0000000000..c9904ca97f --- /dev/null +++ b/src/BenchmarkDotNet/Running/ICategoryDiscoverer.cs @@ -0,0 +1,9 @@ +using System.Reflection; + +namespace BenchmarkDotNet.Running +{ + public interface ICategoryDiscoverer + { + string[] GetCategories(MethodInfo method); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Configs/CategoriesTests.cs b/tests/BenchmarkDotNet.Tests/Configs/CategoriesTests.cs index b742c7af49..28b0310a34 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/CategoriesTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/CategoriesTests.cs @@ -1,5 +1,8 @@ +using System; using System.Linq; +using System.Reflection; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; using BenchmarkDotNet.Running; using Xunit; using Xunit.Abstractions; @@ -15,41 +18,119 @@ public CategoriesTests(ITestOutputHelper output) this.output = output; } - [Fact] - public void CategoryInheritanceTest() + private void Check(params string[] expected) { string Format(BenchmarkCase benchmarkCase) => benchmarkCase.Descriptor.WorkloadMethod.Name + ": " + string.Join("+", benchmarkCase.Descriptor.Categories.OrderBy(category => category)); - var benchmarkCases = BenchmarkConverter - .TypeToBenchmarks(typeof(DerivedClass)) + var actual = BenchmarkConverter + .TypeToBenchmarks(typeof(T)) .BenchmarksCases .OrderBy(x => x.Descriptor.WorkloadMethod.Name) + .Select(Format) .ToList(); - Assert.Equal(2, benchmarkCases.Count); + foreach (string s in actual) + output.WriteLine(s); + + Assert.Equal(expected, actual); + } + + [Fact] + public void CategoryInheritanceTest() => + Check( + "BaseMethod: BaseClassCategory+BaseMethodCategory+DerivedClassCategory", + "DerivedMethod: BaseClassCategory+DerivedClassCategory+DerivedMethodCategory" + ); - output.WriteLine(Format(benchmarkCases[0])); - output.WriteLine(Format(benchmarkCases[1])); + public static class CategoryInheritanceTestScope + { + [BenchmarkCategory("BaseClassCategory")] + public class BaseClass + { + [Benchmark] + [BenchmarkCategory("BaseMethodCategory")] + public void BaseMethod() { } + } - Assert.Equal("BaseMethod: BaseClassCategory+BaseMethodCategory+DerivedClassCategory", Format(benchmarkCases[0])); - Assert.Equal("DerivedMethod: BaseClassCategory+DerivedClassCategory+DerivedMethodCategory", Format(benchmarkCases[1])); + [BenchmarkCategory("DerivedClassCategory")] + public class DerivedClass : BaseClass + { + [Benchmark] + [BenchmarkCategory("DerivedMethodCategory")] + public void DerivedMethod() { } + } } - [BenchmarkCategory("BaseClassCategory")] - public class BaseClass + [Fact] + public void CategoryNoInheritanceTest() => + Check( + "BaseMethod: BaseMethodCategory+DerivedClassCategory", + "DerivedMethod: DerivedClassCategory+DerivedMethodCategory" + ); + + public static class CategoryNoInheritanceTestScope { - [Benchmark] - [BenchmarkCategory("BaseMethodCategory")] - public void BaseMethod() { } + [BenchmarkCategory("BaseClassCategory")] + public class BaseClass + { + [Benchmark] + [BenchmarkCategory("BaseMethodCategory")] + public void BaseMethod() { } + } + + [BenchmarkCategory("DerivedClassCategory")] + [CategoryDiscoverer(false)] + public class DerivedClass : BaseClass + { + [Benchmark] + [BenchmarkCategory("DerivedMethodCategory")] + public void DerivedMethod() { } + } } - [BenchmarkCategory("DerivedClassCategory")] - public class DerivedClass : BaseClass + [Fact] + public void CustomCategoryDiscovererTest() => + Check( + "Aaa: A+PermanentCategory", + "Bbb: B+PermanentCategory" + ); + + public static class CustomCategoryDiscovererTestScope { - [Benchmark] - [BenchmarkCategory("DerivedMethodCategory")] - public void DerivedMethod() { } + private class CustomCategoryDiscoverer : ICategoryDiscoverer + { + public string[] GetCategories(MethodInfo method) + { + return new[] + { + "PermanentCategory", + method.Name.Substring(0, 1) + }; + } + } + + [AttributeUsage(AttributeTargets.Class)] + private class CustomCategoryDiscovererAttribute : Attribute, IConfigSource + { + public CustomCategoryDiscovererAttribute() + { + Config = ManualConfig.CreateEmpty().WithCategoryDiscoverer(new CustomCategoryDiscoverer()); + } + + public IConfig Config { get; } + } + + + [CustomCategoryDiscoverer] + public class Benchmarks + { + [Benchmark] + public void Aaa() { } + + [Benchmark] + public void Bbb() { } + } } } } \ No newline at end of file