Support for custom metrics reported in the Benchmarks (#735)

* simplify and cleanup the code, remove dead code

* use `Target` to specify that given setup method should be executed for selected benchmarks, not all

* consume the result of Predict to make sure it does not get dead-code eliminated

* reference input files from .csproj and copy them to output dir, don't rely on hardcoded folder hierarchy

* every ML.NET benchmark allocates a lot of memory and should be executed in a dedicated process

* make it possible for every type to report different metrics

* enforce current culture as "en-us" because the input data files use dot as decimal separator (and it fails for cultures with ",")

* for our time consuming benchmarks 1 warmup iteration is enough

* workaround for the auto-generated code to avoid name coflict for Microsoft.ML.Runtime.IHost and BenchmarkDotNet.Engines.IHost..

* add comment about why we need a custom toolchain

* update BDN version to allow benchmarking with CoreRun

* code review fix: spacing

Author: adamsitnik

build/Dependencies.props

@@ -10,7 +10,7 @@
     <LightGBMPackageVersion>2.1.2.2</LightGBMPackageVersion>
     <MlNetMklDepsPackageVersion>0.0.0.5</MlNetMklDepsPackageVersion>
     <SystemDrawingCommonPackageVersion>4.5.0</SystemDrawingCommonPackageVersion>
-    <BenchmarkDotNetVersion>0.11.0</BenchmarkDotNetVersion>
+    <BenchmarkDotNetVersion>0.11.1</BenchmarkDotNetVersion>
     <TensorFlowVersion>1.10.0</TensorFlowVersion>
   </PropertyGroup>
 </Project>

test/Microsoft.ML.Benchmarks/Harness/Metrics.cs

@@ -0,0 +1,110 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Microsoft.ML.Benchmarks
+{
+    public abstract class WithExtraMetrics
+    {
+        protected abstract IEnumerable<Metric> GetMetrics();
+
+        /// <summary>
+        ///  this method is executed after running the benchmrks
+        ///  we use it as hack to simply print to console so ExtraMetricColumn can parse the output
+        /// </summary>
+        [GlobalCleanup]
+        public void ReportMetrics()
+        {
+            foreach (var metric in GetMetrics())
+            {
+                Console.WriteLine(metric.ToParsableString());
+            }
+        }
+    }
+
+    public class ExtraMetricColumn : IColumn
+    {
+        public string ColumnName => "Extra Metric";
+        public string Id => nameof(ExtraMetricColumn);
+        public string Legend => "Value of the provided extra metric";
+        public bool IsNumeric => true;
+        public bool IsDefault(Summary summary, BenchmarkCase benchmark) => true;
+        public bool IsAvailable(Summary summary) => true;
+        public bool AlwaysShow => true;
+        public ColumnCategory Category => ColumnCategory.Custom;
+        public int PriorityInCategory => 1;
+        public UnitType UnitType => UnitType.Dimensionless;
+        public string GetValue(Summary summary, BenchmarkCase benchmark) => GetValue(summary, benchmark, null);
+        public override string ToString() => ColumnName;
+
+        public string GetValue(Summary summary, BenchmarkCase benchmark, ISummaryStyle style)
+        {
+            if (!summary.HasReport(benchmark))
+                return "-";
+
+            var results = summary[benchmark].ExecuteResults;
+            if (results.Count != 1)
+                return "-";
+
+            var result = results.Single();
+            var buffer = new StringBuilder();
+
+            foreach (var line in result.ExtraOutput)
+            {
+                if (Metric.TryParse(line, out Metric metric))
+                {
+                    if (buffer.Length > 0)
+                        buffer.Append(", ");
+
+                    buffer.Append(metric.ToColumnValue());
+                }
+            }
+
+            return buffer.Length > 0 ? buffer.ToString() : "-";
+        }
+    }
+
+    public struct Metric
+    {
+        private const string Prefix = "// Metric";
+        private const char Separator = '#';
+
+        public string Name { get; }
+        public string Value { get; }
+
+        public Metric(string name, string value) : this()
+        {
+            Name = name;
+            Value = value;
+        }
+
+        public string ToColumnValue()
+            => $"{Name}: {Value}";
+
+        public string ToParsableString()
+            => $"{Prefix} {Separator} {Name} {Separator} {Value}";
+
+        public static bool TryParse(string line, out Metric metric)
+        {
+            metric = default;
+
+            if (!line.StartsWith(Prefix))
+                return false;
+
+            var splitted = line.Split(Separator);
+
+            metric = new Metric(splitted[1].Trim(), splitted[2].Trim());
+
+            return true;
+        }
+    }
+}

test/Microsoft.ML.Benchmarks/Harness/ProjectGenerator.cs

@@ -0,0 +1,54 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Toolchains.CsProj;
+using System;
+using System.IO;
+using System.Linq;
+
+namespace Microsoft.ML.Benchmarks.Harness
+{
+    /// <summary>
+    /// to avoid side effects of benchmarks affect each other BenchmarkDotNet runs every benchmark in a standalone, dedicated process
+    /// however to do that it needs to be able to create, build and run new executable
+    /// 
+    /// the problem with ML.NET is that it has native dependencies, which are NOT copied by MSBuild to the output folder
+    /// in case where A has native dependency and B references A
+    /// 
+    /// this is why this class exists: to copy the native dependencies to folder with .exe
+    /// </summary>
+    public class ProjectGenerator : CsProjGenerator
+    {
+        public ProjectGenerator(string targetFrameworkMoniker) : base(targetFrameworkMoniker, platform => platform.ToConfig(), null)
+        {
+        }
+
+        protected override void CopyAllRequiredFiles(ArtifactsPaths artifactsPaths)
+        {
+            base.CopyAllRequiredFiles(artifactsPaths);
+
+            CopyMissingNativeDependencies(artifactsPaths);
+        }
+
+        private void CopyMissingNativeDependencies(ArtifactsPaths artifactsPaths)
+        {
+            var foldeWithAutogeneratedExe = Path.GetDirectoryName(artifactsPaths.ExecutablePath);
+            var folderWithNativeDependencies = Path.GetDirectoryName(typeof(ProjectGenerator).Assembly.Location);
+
+            foreach (var nativeDependency in Directory
+                .EnumerateFiles(folderWithNativeDependencies)
+                .Where(fileName => ContainsWithIgnoreCase(fileName, "native")))
+            {
+                File.Copy(
+                    sourceFileName: nativeDependency,
+                    destFileName: Path.Combine(foldeWithAutogeneratedExe, Path.GetFileName(nativeDependency)), 
+                    overwrite: true);
+            }
+        }
+
+        bool ContainsWithIgnoreCase(string text, string word) => text != null && text.IndexOf(word, StringComparison.InvariantCultureIgnoreCase) >= 0;
+    }
+}

test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs

@@ -4,6 +4,7 @@
 
 using BenchmarkDotNet.Attributes;
 using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.Internal.Calibration;
 using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.EntryPoints;
 using Microsoft.ML.Runtime.KMeans;
@@ -13,21 +14,11 @@ namespace Microsoft.ML.Benchmarks
 {
     public class KMeansAndLogisticRegressionBench
     {
-        private static string s_dataPath;
+        private readonly string _dataPath = Program.GetInvariantCultureDataPath("adult.train");
 
         [Benchmark]
-        public IPredictor TrainKMeansAndLR() => TrainKMeansAndLRCore();
-
-        [GlobalSetup]
-        public void Setup()
+        public ParameterMixingCalibratedPredictor TrainKMeansAndLR()
         {
-            s_dataPath = Program.GetDataPath("adult.train");
-        }
-
-        private static IPredictor TrainKMeansAndLRCore()
-        {
-            string dataPath = s_dataPath;
-
             using (var env = new TlcEnvironment(seed: 1))
             {
                 // Pipeline
@@ -53,7 +44,7 @@ private static IPredictor TrainKMeansAndLRCore()
                                     new TextLoader.Range() { Min = 10, Max = 12 }
                                 })
                         }
-                    }, new MultiFileSource(dataPath));
+                    }, new MultiFileSource(_dataPath));
 
                 IDataTransform trans = CategoricalTransform.Create(env, new CategoricalTransform.Arguments
                 {
@@ -83,4 +74,4 @@ private static IPredictor TrainKMeansAndLRCore()
             }
         }
     }
-}
+}

test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj

@@ -20,4 +20,16 @@
   <ItemGroup>
     <NativeAssemblyReference Include="CpuMathNative" />
   </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Input\" />
+    <Content Include="..\data\iris.txt" Link="Input\iris.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <None Include="..\data\adult.train" Link="Input\adult.train">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Include="..\data\wikipedia-detox-250-line-data.tsv" Link="Input\wikipedia-detox-250-line-data.tsv">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
 </Project>

test/Microsoft.ML.Benchmarks/Program.cs

@@ -6,11 +6,13 @@
 using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Running;
-using BenchmarkDotNet.Columns;
-using BenchmarkDotNet.Reports;
-using BenchmarkDotNet.Toolchains.InProcess;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Toolchains.DotNetCli;
+using Microsoft.ML.Benchmarks.Harness;
+using System.Globalization;
 using System.IO;
-using Microsoft.ML.Models;
+using System.Threading;
 
 namespace Microsoft.ML.Benchmarks
 {
@@ -28,52 +30,33 @@ static void Main(string[] args)
         private static IConfig CreateCustomConfig() 
             => DefaultConfig.Instance
                 .With(Job.Default
+                    .WithWarmupCount(1) // for our time consuming benchmarks 1 warmup iteration is enough
                     .WithMaxIterationCount(20)
-                    .With(InProcessToolchain.Instance))
-                .With(new ClassificationMetricsColumn("AccuracyMacro", "Macro-average accuracy of the model"))
+                    .With(CreateToolchain()))
+                .With(new ExtraMetricColumn())
                 .With(MemoryDiagnoser.Default);
 
-        internal static string GetDataPath(string name)
-            => Path.GetFullPath(Path.Combine(_dataRoot, name));
-
-        static readonly string _dataRoot;
-        static Program()
+        /// <summary>
+        /// we need our own toolchain because MSBuild by default does not copy recursive native dependencies to the output
+        /// </summary>
+        private static IToolchain CreateToolchain()
         {
-            var currentAssemblyLocation = new FileInfo(typeof(Program).Assembly.Location);
-            var rootDir = currentAssemblyLocation.Directory.Parent.Parent.Parent.Parent.FullName;
-            _dataRoot = Path.Combine(rootDir, "test", "data");
+            var csProj = CsProjCoreToolchain.Current.Value;
+            var tfm = NetCoreAppSettings.Current.Value.TargetFrameworkMoniker;
+
+            return new Toolchain(
+                tfm, 
+                new ProjectGenerator(tfm), 
+                csProj.Builder, 
+                csProj.Executor);
         }
-    }
-
-    public class ClassificationMetricsColumn : IColumn
-    {
-        private readonly string _metricName;
-        private readonly string _legend;
 
-        public ClassificationMetricsColumn(string metricName, string legend)
+        internal static string GetInvariantCultureDataPath(string name)
         {
-            _metricName = metricName;
-            _legend = legend;
-        }
-
-        public string ColumnName => _metricName;
-        public string Id => _metricName;
-        public string Legend => _legend;
-        public bool IsNumeric => true;
-        public bool IsDefault(Summary summary, BenchmarkCase benchmark) => true;
-        public bool IsAvailable(Summary summary) => true;
-        public bool AlwaysShow => true;
-        public ColumnCategory Category => ColumnCategory.Custom;
-        public int PriorityInCategory => 1;
-        public UnitType UnitType => UnitType.Dimensionless;
+            // enforce Neutral Language as "en-us" because the input data files use dot as decimal separator (and it fails for cultures with ",")
+            Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
 
-        public string GetValue(Summary summary, BenchmarkCase benchmark, ISummaryStyle style)
-        {
-            var property = typeof(ClassificationMetrics).GetProperty(_metricName);
-            return property.GetValue(StochasticDualCoordinateAscentClassifierBench.s_metrics).ToString();
+            return Path.Combine(Path.GetDirectoryName(typeof(Program).Assembly.Location), "Input", name);
         }
-        public string GetValue(Summary summary, BenchmarkCase benchmark) => GetValue(summary, benchmark, null);
-
-        public override string ToString() => ColumnName;
     }
 }