AutoFit return type is now an IEnumerable (dotnet#55)

AutoFit returns is now an IEnumerable - this enables many good things

Implementing variety of early stopping criteria (See sample)
Early discard of models that are no good. This improves memory usage efficiency. (See sample)
No need to implement a callback to get results back
Getting best score is now outside of API implementation. It is a simple math function to compare scores (See sample).

Also templatized the return type for better type safety through out the code.

Author: vinodshanbhag

src/AutoML/API/MLContextAutoFitExtensions.cs

@@ -12,32 +12,28 @@ namespace Microsoft.ML.Auto
 {
     public static class RegressionExtensions
     {
-        public static RegressionResult AutoFit(this RegressionContext context,
+        public static IEnumerable<IterationResult<RegressionMetrics>> AutoFit(this RegressionContext context,
             IDataView trainData,
             string label = DefaultColumnNames.Label,
             IDataView validationData = null,
             uint timeoutInMinutes = AutoFitDefaults.TimeOutInMinutes,
             IEstimator<ITransformer> preFeaturizers = null,
-            IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
-            CancellationToken cancellationToken = default,
-            IProgress<RegressionIterationResult> iterationCallback = null)
+            IEnumerable<(string, ColumnPurpose)> columnPurposes = null)
         {
             var settings = new AutoFitSettings();
             settings.StoppingCriteria.TimeOutInMinutes = timeoutInMinutes;
 
             return AutoFit(context, trainData, label, validationData, settings,
-                preFeaturizers, columnPurposes, cancellationToken, iterationCallback, null);
+                preFeaturizers, columnPurposes, null);
         }
 
-        internal static RegressionResult AutoFit(this RegressionContext context,
+        internal static IEnumerable<IterationResult<RegressionMetrics>> AutoFit(this RegressionContext context,
             IDataView trainData,
             string label = DefaultColumnNames.Label,
             IDataView validationData = null,
             AutoFitSettings settings = null,
             IEstimator<ITransformer> preFeaturizers = null,
             IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
-            CancellationToken cancellationToken = default,
-            IProgress<RegressionIterationResult> iterationCallback = null,
             IDebugLogger debugLogger = null)
         {
             UserInputValidationUtil.ValidateAutoFitArgs(trainData, label, validationData, settings, columnPurposes);
@@ -48,49 +44,38 @@ internal static RegressionResult AutoFit(this RegressionContext context,
             }
 
             // run autofit & get all pipelines run in that process
-            var (allPipelines, bestPipeline) = AutoFitApi.Fit(trainData, validationData, label,
-                settings, preFeaturizers, TaskKind.Regression, OptimizingMetric.RSquared, columnPurposes, debugLogger);
+            var autoFitter = new AutoFitter<RegressionMetrics>(TaskKind.Regression, trainData, label, validationData,
+                settings, preFeaturizers, columnPurposes,
+                OptimizingMetric.RSquared, debugLogger);
 
-            var results = new RegressionIterationResult[allPipelines.Length];
-            for (var i = 0; i < results.Length; i++)
-            {
-                var iterationResult = allPipelines[i];
-                var result = new RegressionIterationResult(iterationResult.Model, (RegressionMetrics)iterationResult.EvaluatedMetrics, iterationResult.ScoredValidationData, iterationResult.Pipeline.ToPipeline());
-                results[i] = result;
-            }
-            var bestResult = new RegressionIterationResult(bestPipeline.Model, (RegressionMetrics)bestPipeline.EvaluatedMetrics, bestPipeline.ScoredValidationData, bestPipeline.Pipeline.ToPipeline());
-            return new RegressionResult(bestResult, results);
+            return autoFitter.Fit();
         }
     }
 
     public static class BinaryClassificationExtensions
     {
-        public static BinaryClassificationResult AutoFit(this BinaryClassificationContext context,
+        public static IEnumerable<IterationResult<BinaryClassificationMetrics>> AutoFit(this BinaryClassificationContext context,
             IDataView trainData,
             string label = DefaultColumnNames.Label,
             IDataView validationData = null,
             uint timeoutInMinutes = AutoFitDefaults.TimeOutInMinutes,
             IEstimator<ITransformer> preFeaturizers = null,
-            IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
-            CancellationToken cancellationToken = default,
-            IProgress<BinaryClassificationItertionResult> iterationCallback = null)
+            IEnumerable<(string, ColumnPurpose)> columnPurposes = null)
         {
             var settings = new AutoFitSettings();
             settings.StoppingCriteria.TimeOutInMinutes = timeoutInMinutes;
 
             return AutoFit(context, trainData, label, validationData, settings,
-                preFeaturizers, columnPurposes, cancellationToken, iterationCallback, null);
+                preFeaturizers, columnPurposes, null);
         }
 
-        internal static BinaryClassificationResult AutoFit(this BinaryClassificationContext context,
+        internal static IEnumerable<IterationResult<BinaryClassificationMetrics>> AutoFit(this BinaryClassificationContext context,
             IDataView trainData,
             string label = DefaultColumnNames.Label,
             IDataView validationData = null,
             AutoFitSettings settings = null,
             IEstimator<ITransformer> preFeaturizers = null,
             IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
-            CancellationToken cancellationToken = default,
-            IProgress<BinaryClassificationItertionResult> iterationCallback = null,
             IDebugLogger debugLogger = null)
         {
             UserInputValidationUtil.ValidateAutoFitArgs(trainData, label, validationData, settings, columnPurposes);
@@ -101,159 +86,67 @@ internal static BinaryClassificationResult AutoFit(this BinaryClassificationCont
             }
 
             // run autofit & get all pipelines run in that process
-            var (allPipelines, bestPipeline) = AutoFitApi.Fit(trainData, validationData, label,
-                settings, preFeaturizers, TaskKind.BinaryClassification, OptimizingMetric.Accuracy,
-                columnPurposes, debugLogger);
-
-            var results = new BinaryClassificationItertionResult[allPipelines.Length];
-            for (var i = 0; i < results.Length; i++)
-            {
-                var iterationResult = allPipelines[i];
-                var result = new BinaryClassificationItertionResult(iterationResult.Model, (BinaryClassificationMetrics)iterationResult.EvaluatedMetrics, iterationResult.ScoredValidationData, iterationResult.Pipeline.ToPipeline());
-                results[i] = result;
-            }
-            var bestResult = new BinaryClassificationItertionResult(bestPipeline.Model, (BinaryClassificationMetrics)bestPipeline.EvaluatedMetrics, bestPipeline.ScoredValidationData, bestPipeline.Pipeline.ToPipeline());
-            return new BinaryClassificationResult(bestResult, results);
+            var autoFitter = new AutoFitter<BinaryClassificationMetrics>(TaskKind.BinaryClassification, trainData, label, validationData,
+                settings, preFeaturizers, columnPurposes, 
+                OptimizingMetric.RSquared, debugLogger);
+            
+            return autoFitter.Fit();
         }
     }
 
     public static class MulticlassExtensions
     {
-        public static MulticlassClassificationResult AutoFit(this MulticlassClassificationContext context,
+        public static IEnumerable<IterationResult<MultiClassClassifierMetrics>> AutoFit(this MulticlassClassificationContext context,
             IDataView trainData,
             string label = DefaultColumnNames.Label,
             IDataView validationData = null,
             uint timeoutInMinutes = AutoFitDefaults.TimeOutInMinutes,
             IEstimator<ITransformer> preFeaturizers = null,
-            IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
-            CancellationToken cancellationToken = default,
-            IProgress<MulticlassClassificationIterationResult> iterationCallback = null)
+            IEnumerable<(string, ColumnPurpose)> columnPurposes = null)
         {
             var settings = new AutoFitSettings();
             settings.StoppingCriteria.TimeOutInMinutes = timeoutInMinutes;
 
             return AutoFit(context, trainData, label, validationData, settings,
-                preFeaturizers, columnPurposes, cancellationToken, iterationCallback, null);
+                preFeaturizers, columnPurposes, null);
         }
 
-        internal static MulticlassClassificationResult AutoFit(this MulticlassClassificationContext context,
+        internal static IEnumerable<IterationResult<MultiClassClassifierMetrics>> AutoFit(this MulticlassClassificationContext context,
             IDataView trainData,
             string label = DefaultColumnNames.Label,
             IDataView validationData = null,
             AutoFitSettings settings = null,
             IEstimator<ITransformer> preFeaturizers = null,
             IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
-            CancellationToken cancellationToken = default,
-            IProgress<MulticlassClassificationIterationResult> iterationCallback = null, IDebugLogger debugLogger = null)
+            IDebugLogger debugLogger = null)
         {
             UserInputValidationUtil.ValidateAutoFitArgs(trainData, label, validationData, settings, columnPurposes);
 
             if (validationData == null)
             {
                 (trainData, validationData) = context.TestValidateSplit(trainData);
             }
-
+            
             // run autofit & get all pipelines run in that process
-            var (allPipelines, bestPipeline) = AutoFitApi.Fit(trainData, validationData, label,
-                settings, preFeaturizers, TaskKind.MulticlassClassification, OptimizingMetric.Accuracy,
-                columnPurposes, debugLogger);
-
-            var results = new MulticlassClassificationIterationResult[allPipelines.Length];
-            for (var i = 0; i < results.Length; i++)
-            {
-                var iterationResult = allPipelines[i];
-                var result = new MulticlassClassificationIterationResult(iterationResult.Model, (MultiClassClassifierMetrics)iterationResult.EvaluatedMetrics, iterationResult.ScoredValidationData, iterationResult.Pipeline.ToPipeline());
-                results[i] = result;
-            }
-            var bestResult = new MulticlassClassificationIterationResult(bestPipeline.Model, (MultiClassClassifierMetrics)bestPipeline.EvaluatedMetrics, bestPipeline.ScoredValidationData, bestPipeline.Pipeline.ToPipeline());
-            return new MulticlassClassificationResult(bestResult, results);
-        }
-    }
-
-    public class BinaryClassificationResult
-    {
-        public readonly BinaryClassificationItertionResult BestIteration;
-        public readonly BinaryClassificationItertionResult[] IterationResults;
-
-        public BinaryClassificationResult(BinaryClassificationItertionResult bestPipeline,
-            BinaryClassificationItertionResult[] iterationResults)
-        {
-            BestIteration = bestPipeline;
-            IterationResults = iterationResults;
-        }
-    }
-
-    public class MulticlassClassificationResult
-    {
-        public readonly MulticlassClassificationIterationResult BestIteration;
-        public readonly MulticlassClassificationIterationResult[] IterationResults;
-
-        public MulticlassClassificationResult(MulticlassClassificationIterationResult bestPipeline,
-            MulticlassClassificationIterationResult[] iterationResults)
-        {
-            BestIteration = bestPipeline;
-            IterationResults = iterationResults;
-        }
-    }
-
-    public class RegressionResult
-    {
-        public readonly RegressionIterationResult BestIteration;
-        public readonly RegressionIterationResult[] IterationResults;
-
-        public RegressionResult(RegressionIterationResult bestPipeline,
-            RegressionIterationResult[] iterationResults)
-        {
-            BestIteration = bestPipeline;
-            IterationResults = iterationResults;
-        }
-    }
-
-    public class BinaryClassificationItertionResult
-    {
-        public readonly BinaryClassificationMetrics Metrics;
-        public readonly ITransformer Model;
-        public readonly IDataView ScoredValidationData;
-        internal readonly Pipeline Pipeline;
-
-        internal BinaryClassificationItertionResult(ITransformer model, BinaryClassificationMetrics metrics, IDataView scoredValidationData, Pipeline pipeline)
-        {
-            Model = model;
-            ScoredValidationData = scoredValidationData;
-            Metrics = metrics;
-            Pipeline = pipeline;
-        }
-    }
-
-    public class MulticlassClassificationIterationResult
-    {
-        public readonly MultiClassClassifierMetrics Metrics;
-        public readonly ITransformer Model;
-        public readonly IDataView ScoredValidationData;
-        internal readonly Pipeline Pipeline;
-
-        internal MulticlassClassificationIterationResult(ITransformer model, MultiClassClassifierMetrics metrics, IDataView scoredValidationData, Pipeline pipeline)
-        {
-            Model = model;
-            Metrics = metrics;
-            ScoredValidationData = scoredValidationData;
-            Pipeline = pipeline;
+            var autoFitter = new AutoFitter<MultiClassClassifierMetrics>(TaskKind.MulticlassClassification, trainData, label, validationData,
+                settings, preFeaturizers, columnPurposes, OptimizingMetric.RSquared, debugLogger);
+            return autoFitter.Fit();
         }
     }
 
-    public class RegressionIterationResult
+    public class IterationResult<T>
     {
-        public readonly RegressionMetrics Metrics;
+        public readonly T Metrics;
         public readonly ITransformer Model;
-        public readonly IDataView ScoredValidationData;
+        public readonly Exception Exception;
         internal readonly Pipeline Pipeline;
 
-        internal RegressionIterationResult(ITransformer model, RegressionMetrics metrics, IDataView scoredValidationData, Pipeline pipeline)
+        internal IterationResult(ITransformer model, T metrics, Pipeline pipeline, Exception exception)
         {
             Model = model;
             Metrics = metrics;
-            ScoredValidationData = scoredValidationData;
             Pipeline = pipeline;
+            Exception = exception;
         }
     }
 }

src/AutoML/API/Pipeline.cs

@@ -23,7 +23,7 @@ internal Pipeline()
 
         public IEstimator<ITransformer> ToEstimator()
         {
-            var inferredPipeline = InferredPipeline.FromPipeline(this);
+            var inferredPipeline = SuggestedPipeline.FromPipeline(this);
             return inferredPipeline.ToEstimator();
         }
     }
@@ -87,7 +87,7 @@ internal CustomProperty()
         }
     }
 
-    internal class PipelineRunResult
+    internal class PipelineScore
     {
         public readonly double Score;
 
@@ -99,7 +99,7 @@ internal class PipelineRunResult
 
         internal readonly Pipeline Pipeline;
 
-        internal PipelineRunResult(Pipeline pipeline, double score, bool runSucceeded)
+        internal PipelineScore(Pipeline pipeline, double score, bool runSucceeded)
         {
             Pipeline = pipeline;
             Score = score;