From 8075f39858844e77645e5ae73880f73813083138 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 16 Feb 2023 12:17:09 -0800 Subject: [PATCH 1/3] fix #6570 --- .../API/BinaryClassificationExperiment.cs | 38 ++++++++++--------- .../API/MulticlassClassificationExperiment.cs | 34 ++++++++--------- .../API/RegressionExperiment.cs | 35 ++++++++--------- 3 files changed, 51 insertions(+), 56 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 70db9035e5..6998193382 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -365,6 +365,7 @@ public TrialResult Run(TrialSettings settings) { var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; var pipeline = _pipeline.BuildFromOption(_context, parameter); + if (_datasetManager is ICrossValidateDatasetManager datasetManager) { var stopWatch = new Stopwatch(); @@ -375,14 +376,8 @@ public TrialResult Run(TrialSettings settings) // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. var res = metrics[_rnd.Next(fold)]; var model = res.Model; - var metric = metricManager.Metric switch - { - BinaryClassificationMetric.PositivePrecision => res.Metrics.PositivePrecision, - BinaryClassificationMetric.Accuracy => res.Metrics.Accuracy, - BinaryClassificationMetric.AreaUnderRocCurve => res.Metrics.AreaUnderRocCurve, - BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => res.Metrics.AreaUnderPrecisionRecallCurve, - _ => throw new NotImplementedException($"{metricManager.MetricName} is not supported!"), - }; + var metric = GetMetric(metricManager.Metric, res.Metrics); + var loss = metricManager.IsMaximize ? -metric : metric; stopWatch.Stop(); @@ -407,16 +402,7 @@ public TrialResult Run(TrialSettings settings) var model = pipeline.Fit(trainTestDatasetManager.TrainDataset); var eval = model.Transform(trainTestDatasetManager.TestDataset); var metrics = _context.BinaryClassification.EvaluateNonCalibrated(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn); - - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var metric = Enum.Parse(typeof(BinaryClassificationMetric), metricManager.MetricName) switch - { - BinaryClassificationMetric.PositivePrecision => metrics.PositivePrecision, - BinaryClassificationMetric.Accuracy => metrics.Accuracy, - BinaryClassificationMetric.AreaUnderRocCurve => metrics.AreaUnderRocCurve, - BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metrics.AreaUnderPrecisionRecallCurve, - _ => throw new NotImplementedException($"{metricManager.Metric} is not supported!"), - }; + var metric = GetMetric(metricManager.Metric, metrics); var loss = metricManager.IsMaximize ? -metric : metric; stopWatch.Stop(); @@ -459,5 +445,21 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) throw; } } + + private double GetMetric(BinaryClassificationMetric metric, BinaryClassificationMetrics metrics) + { + return metric switch + { + BinaryClassificationMetric.PositivePrecision => metrics.PositivePrecision, + BinaryClassificationMetric.Accuracy => metrics.Accuracy, + BinaryClassificationMetric.AreaUnderRocCurve => metrics.AreaUnderRocCurve, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metrics.AreaUnderPrecisionRecallCurve, + BinaryClassificationMetric.PositiveRecall => metrics.PositiveRecall, + BinaryClassificationMetric.NegativePrecision => metrics.NegativePrecision, + BinaryClassificationMetric.NegativeRecall => metrics.NegativeRecall, + BinaryClassificationMetric.F1Score => metrics.F1Score, + _ => throw new NotImplementedException($"{metric} is not supported!"), + }; + } } } diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index df96c28873..d711f45043 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -371,15 +371,7 @@ public TrialResult Run(TrialSettings settings) // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. var res = metrics[_rnd.Next(fold)]; var model = res.Model; - var metric = metricManager.Metric switch - { - MulticlassClassificationMetric.MacroAccuracy => res.Metrics.MacroAccuracy, - MulticlassClassificationMetric.MicroAccuracy => res.Metrics.MicroAccuracy, - MulticlassClassificationMetric.LogLoss => res.Metrics.LogLoss, - MulticlassClassificationMetric.LogLossReduction => res.Metrics.LogLossReduction, - MulticlassClassificationMetric.TopKAccuracy => res.Metrics.TopKAccuracy, - _ => throw new NotImplementedException($"{metricManager.MetricName} is not supported!"), - }; + var metric = GetMetric(metricManager.Metric, res.Metrics); var loss = metricManager.IsMaximize ? -metric : metric; stopWatch.Stop(); @@ -405,16 +397,7 @@ public TrialResult Run(TrialSettings settings) var model = pipeline.Fit(trainTestDatasetManager.TrainDataset); var eval = model.Transform(trainTestDatasetManager.TestDataset); var metrics = _context.MulticlassClassification.Evaluate(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn); - - var metric = metricManager.Metric switch - { - MulticlassClassificationMetric.MacroAccuracy => metrics.MacroAccuracy, - MulticlassClassificationMetric.MicroAccuracy => metrics.MicroAccuracy, - MulticlassClassificationMetric.LogLoss => metrics.LogLoss, - MulticlassClassificationMetric.LogLossReduction => metrics.LogLossReduction, - MulticlassClassificationMetric.TopKAccuracy => metrics.TopKAccuracy, - _ => throw new NotImplementedException($"{metricManager.Metric} is not supported!"), - }; + var metric = GetMetric(metricManager.Metric, metrics); var loss = metricManager.IsMaximize ? -metric : metric; stopWatch.Stop(); @@ -458,6 +441,19 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) } } + private double GetMetric(MulticlassClassificationMetric metric, MulticlassClassificationMetrics metrics) + { + return metric switch + { + MulticlassClassificationMetric.MacroAccuracy => metrics.MacroAccuracy, + MulticlassClassificationMetric.MicroAccuracy => metrics.MicroAccuracy, + MulticlassClassificationMetric.LogLoss => metrics.LogLoss, + MulticlassClassificationMetric.LogLossReduction => metrics.LogLossReduction, + MulticlassClassificationMetric.TopKAccuracy => metrics.TopKAccuracy, + _ => throw new NotImplementedException($"{metric} is not supported!"), + }; + } + public void Dispose() { _context.CancelExecution(); diff --git a/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs b/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs index e8ac5e405e..751c813028 100644 --- a/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs @@ -398,14 +398,7 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. var res = metrics[_rnd.Next(fold)]; var model = res.Model; - var metric = metricManager.Metric switch - { - RegressionMetric.RootMeanSquaredError => res.Metrics.RootMeanSquaredError, - RegressionMetric.RSquared => res.Metrics.RSquared, - RegressionMetric.MeanSquaredError => res.Metrics.MeanSquaredError, - RegressionMetric.MeanAbsoluteError => res.Metrics.MeanAbsoluteError, - _ => throw new NotImplementedException($"{metricManager.MetricName} is not supported!"), - }; + var metric = GetMetric(metricManager.Metric, res.Metrics); var loss = metricManager.IsMaximize ? -metric : metric; stopWatch.Stop(); @@ -430,16 +423,8 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) stopWatch.Start(); var model = pipeline.Fit(trainTestDatasetManager.TrainDataset); var eval = model.Transform(trainTestDatasetManager.TestDataset); - var res = _context.Regression.Evaluate(eval, metricManager.LabelColumn, scoreColumnName: metricManager.ScoreColumn); - - var metric = metricManager.Metric switch - { - RegressionMetric.RootMeanSquaredError => res.RootMeanSquaredError, - RegressionMetric.RSquared => res.RSquared, - RegressionMetric.MeanSquaredError => res.MeanSquaredError, - RegressionMetric.MeanAbsoluteError => res.MeanAbsoluteError, - _ => throw new NotImplementedException($"{metricManager.Metric} is not supported!"), - }; + var metrics = _context.Regression.Evaluate(eval, metricManager.LabelColumn, scoreColumnName: metricManager.ScoreColumn); + var metric = GetMetric(metricManager.Metric, metrics); var loss = metricManager.IsMaximize ? -metric : metric; stopWatch.Stop(); @@ -452,7 +437,7 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - Metrics = res, + Metrics = metrics, Pipeline = pipeline, } as TrialResult); } @@ -476,5 +461,17 @@ public void Dispose() _context.CancelExecution(); _context = null; } + + private double GetMetric(RegressionMetric metric, RegressionMetrics metrics) + { + return metric switch + { + RegressionMetric.RootMeanSquaredError => metrics.RootMeanSquaredError, + RegressionMetric.RSquared => metrics.RSquared, + RegressionMetric.MeanSquaredError => metrics.MeanSquaredError, + RegressionMetric.MeanAbsoluteError => metrics.MeanAbsoluteError, + _ => throw new NotImplementedException($"{metric} is not supported!"), + }; + } } } From 1aacc932fe63846e50600c5ea8bd0d5bdf48285c Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Mon, 6 Mar 2023 16:44:30 -0800 Subject: [PATCH 2/3] fix build error --- src/Microsoft.ML.AutoML/API/RegressionExperiment.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs b/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs index 2d671ce7f1..4904a990fd 100644 --- a/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs @@ -441,7 +441,6 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - Metrics = res, Pipeline = refitPipeline, } as TrialResult); } From 0b19772cdf142bf1fa7543f76f23f721493ba4be Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Wed, 8 Mar 2023 11:11:22 -0800 Subject: [PATCH 3/3] fix build error --- src/Microsoft.ML.AutoML/API/RegressionExperiment.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs b/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs index 4904a990fd..3adad3fddc 100644 --- a/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs @@ -438,6 +438,7 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) { Loss = loss, Metric = metric, + Metrics = metrics, Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds,