Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use Timer and ctx.CancelExecution() to fix AutoML max-time experiment bug #5445

Merged
merged 29 commits into from
Nov 3, 2020
Merged
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
d0f7054
Use ctx.CalncelExecution() to fix AutoML max-time experiment bug
mstfbl Oct 21, 2020
4fa26f8
Added unit test for checking canceled experiment
mstfbl Oct 21, 2020
48a6267
Nit fix
mstfbl Oct 21, 2020
f324030
Different run time on Linux
mstfbl Oct 22, 2020
ee70024
Review
mstfbl Oct 22, 2020
36bf24e
Testing four ouput
mstfbl Oct 22, 2020
d5d23de
Used reflection to test for contexts being canceled
mstfbl Oct 23, 2020
33cf5a6
Reviews
mstfbl Oct 26, 2020
bfc93e9
Merge remote-tracking branch 'upstream/master' into issue5437
mstfbl Oct 26, 2020
c69a19f
Reviews
mstfbl Oct 28, 2020
299b05b
Added main MLContext listener-timer
mstfbl Oct 29, 2020
2e2d441
Merge remote-tracking branch 'upstream/master' into issue5437
mstfbl Oct 29, 2020
ce747fb
Added PRNG on _context, held onto timers for avoiding GC
mstfbl Oct 30, 2020
7635500
Addressed reviews
mstfbl Oct 30, 2020
94a80de
Unit test edits
mstfbl Oct 30, 2020
abe1d7f
Increase run time of experiment to guarantee probabilities
mstfbl Oct 30, 2020
9585a50
Edited unit test to check produced schema of next run model's predict…
mstfbl Oct 30, 2020
1ab662f
Remove scheme check as different CI builds result in varying schemas
mstfbl Oct 30, 2020
bc9e578
Decrease max experiment time unit test time
mstfbl Oct 30, 2020
71ebf23
Merged with master
mstfbl Oct 31, 2020
2d8d06f
Added Timers
mstfbl Nov 2, 2020
490d8c1
Increase second timer time, edit unit test
mstfbl Nov 2, 2020
b0de1d3
Added try catch for OperationCanceledException in Execute()
mstfbl Nov 3, 2020
0918afa
Add AggregateException try catch to slow unit tests for parallel testing
mstfbl Nov 3, 2020
0922aed
Reviews
mstfbl Nov 3, 2020
ef4b34f
Final reviews
mstfbl Nov 3, 2020
b4b49ce
Added LightGBMFact to binary classification test
mstfbl Nov 3, 2020
6502fc8
Removed extra Operation Stopped exception try catch
mstfbl Nov 3, 2020
28e2f2e
Add back OperationCanceledException to Experiment.cs
mstfbl Nov 3, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add AggregateException try catch to slow unit tests for parallel testing
mstfbl committed Nov 3, 2020
commit 0918afa0da9fcd3bee058dea6f96114ddc6632c4
92 changes: 64 additions & 28 deletions test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading;
@@ -149,6 +150,22 @@ public void AutoFitRegressionTest(string culture)
// If the below assertion fails, increase the experiment time so the number of iterations is met
Assert.True(culture == "en-US" || result.RunDetails.Count() >= 75, $"RunDetails.Count() = {result.RunDetails.Count()}, below 75");
}
catch (AggregateException ae)
{
// During CI unit testing, the host machines can run slower than normal, which
// can increase the run time of unit tests and throw OperationCanceledExceptions
// from multiple threads in the form of a single AggregateException.
foreach (var ex in ae.Flatten().InnerExceptions)
{
var ignoredExceptions = new List<Exception>();
if (ex is OperationCanceledException)
continue;
else
ignoredExceptions.Add(ex);
if (ignoredExceptions.Count > 0)
throw new AggregateException(ignoredExceptions);
}
}
finally
{
Thread.CurrentThread.CurrentCulture = originalCulture;
@@ -268,34 +285,53 @@ public void AutoFitRecommendationTest()
var testDataView = reader.Load(new MultiFileSource(GetDataPath(TestDatasets.trivialMatrixFactorization.testFilename)));

// STEP 2: Run AutoML experiment
ExperimentResult<RegressionMetrics> experimentResult = mlContext.Auto()
.CreateRecommendationExperiment(5)
.Execute(trainDataView, testDataView,
new ColumnInformation()
{
LabelColumnName = labelColumnName,
UserIdColumnName = userColumnName,
ItemIdColumnName = itemColumnName
});

RunDetail<RegressionMetrics> bestRun = experimentResult.BestRun;
Assert.True(experimentResult.RunDetails.Count() > 1);
Assert.NotNull(bestRun.ValidationMetrics);
Assert.True(experimentResult.RunDetails.Max(i => i?.ValidationMetrics?.RSquared* i?.ValidationMetrics?.RSquared) > 0.5);

var outputSchema = bestRun.Model.GetOutputSchema(trainDataView.Schema);
var expectedOutputNames = new string[] { labelColumnName, userColumnName, userColumnName, itemColumnName, itemColumnName, scoreColumnName };
foreach (var col in outputSchema)
Assert.True(col.Name == expectedOutputNames[col.Index]);

IDataView testDataViewWithBestScore = bestRun.Model.Transform(testDataView);
// Retrieve label column's index from the test IDataView
testDataView.Schema.TryGetColumnIndex(labelColumnName, out int labelColumnId);
// Retrieve score column's index from the IDataView produced by the trained model
testDataViewWithBestScore.Schema.TryGetColumnIndex(scoreColumnName, out int scoreColumnId);

var metrices = mlContext.Recommendation().Evaluate(testDataViewWithBestScore, labelColumnName: labelColumnName, scoreColumnName: scoreColumnName);
Assert.NotEqual(0, metrices.MeanSquaredError);
try
{
ExperimentResult<RegressionMetrics> experimentResult = mlContext.Auto()
.CreateRecommendationExperiment(5)
.Execute(trainDataView, testDataView,
new ColumnInformation()
{
LabelColumnName = labelColumnName,
UserIdColumnName = userColumnName,
ItemIdColumnName = itemColumnName
});

RunDetail<RegressionMetrics> bestRun = experimentResult.BestRun;
Assert.True(experimentResult.RunDetails.Count() > 1);
Assert.NotNull(bestRun.ValidationMetrics);
Assert.True(experimentResult.RunDetails.Max(i => i?.ValidationMetrics?.RSquared* i?.ValidationMetrics?.RSquared) > 0.5);

var outputSchema = bestRun.Model.GetOutputSchema(trainDataView.Schema);
var expectedOutputNames = new string[] { labelColumnName, userColumnName, userColumnName, itemColumnName, itemColumnName, scoreColumnName };
foreach (var col in outputSchema)
Assert.True(col.Name == expectedOutputNames[col.Index]);

IDataView testDataViewWithBestScore = bestRun.Model.Transform(testDataView);
// Retrieve label column's index from the test IDataView
testDataView.Schema.TryGetColumnIndex(labelColumnName, out int labelColumnId);
// Retrieve score column's index from the IDataView produced by the trained model
testDataViewWithBestScore.Schema.TryGetColumnIndex(scoreColumnName, out int scoreColumnId);

var metrices = mlContext.Recommendation().Evaluate(testDataViewWithBestScore, labelColumnName: labelColumnName, scoreColumnName: scoreColumnName);
Assert.NotEqual(0, metrices.MeanSquaredError);
}
catch (AggregateException ae)
{
// During CI unit testing, the host machines can run slower than normal, which
// can increase the run time of unit tests and throw OperationCanceledExceptions
// from multiple threads in the form of a single AggregateException.
foreach (var ex in ae.Flatten().InnerExceptions)
{
var ignoredExceptions = new List<Exception>();
if (ex is OperationCanceledException)
continue;
else
ignoredExceptions.Add(ex);
if (ignoredExceptions.Count > 0)
throw new AggregateException(ignoredExceptions);
}
}
}

[Fact]