From 87f0c4b8098813cedfb1deb310af36c6d7b42243 Mon Sep 17 00:00:00 2001 From: Ross Smith Date: Fri, 15 Jun 2018 17:32:55 +0100 Subject: [PATCH 1/2] WIP Propose Sweeps --- Microsoft.ML.sln | 7 ++++ src/Microsoft.ML.Sweeper/Algorithms/Grid.cs | 2 +- .../Microsoft.ML.Sweeper.Tests.csproj | 21 ++++++++++++ .../Microsoft.ML.Sweeper.Tests/SweeperTest.cs | 32 +++++++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 test/Microsoft.ML.Sweeper.Tests/Microsoft.ML.Sweeper.Tests.csproj create mode 100644 test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 3529c0e5b7..436aedcb0b 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -106,6 +106,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Maml", "src\Mi EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Console", "src\Microsoft.ML.Console\Microsoft.ML.Console.csproj", "{362A98CF-FBF7-4EBB-A11B-990BBF845B15}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Sweeper.Tests", "test\Microsoft.ML.Sweeper.Tests\Microsoft.ML.Sweeper.Tests.csproj", "{3DEB504D-7A07-48CE-91A2-8047461CB3D4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -208,6 +210,10 @@ Global {362A98CF-FBF7-4EBB-A11B-990BBF845B15}.Debug|Any CPU.Build.0 = Debug|Any CPU {362A98CF-FBF7-4EBB-A11B-990BBF845B15}.Release|Any CPU.ActiveCfg = Release|Any CPU {362A98CF-FBF7-4EBB-A11B-990BBF845B15}.Release|Any CPU.Build.0 = Release|Any CPU + {3DEB504D-7A07-48CE-91A2-8047461CB3D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3DEB504D-7A07-48CE-91A2-8047461CB3D4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3DEB504D-7A07-48CE-91A2-8047461CB3D4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3DEB504D-7A07-48CE-91A2-8047461CB3D4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -243,6 +249,7 @@ Global {7A9DB75F-2CA5-4184-9EF5-1F17EB39483F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} {64F40A0D-D4C2-4AA7-8470-E9CC437827E4} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {362A98CF-FBF7-4EBB-A11B-990BBF845B15} = {09EADF06-BE25-4228-AB53-95AE3E15B530} + {3DEB504D-7A07-48CE-91A2-8047461CB3D4} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs b/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs index 734cf93e30..aeb66137fb 100644 --- a/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs +++ b/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs @@ -64,7 +64,7 @@ protected SweeperBase(ArgumentsBase args, IHostEnvironment env, IValueGenerator[ SweepParameters = sweepParameters; } - public virtual ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable previousRuns) + public virtual ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable previousRuns = null) { var prevParamSets = previousRuns?.Select(r => r.ParameterSet).ToList() ?? new List(); var result = new List(); diff --git a/test/Microsoft.ML.Sweeper.Tests/Microsoft.ML.Sweeper.Tests.csproj b/test/Microsoft.ML.Sweeper.Tests/Microsoft.ML.Sweeper.Tests.csproj new file mode 100644 index 0000000000..59a9a3221e --- /dev/null +++ b/test/Microsoft.ML.Sweeper.Tests/Microsoft.ML.Sweeper.Tests.csproj @@ -0,0 +1,21 @@ + + + + netcoreapp2.0 + + false + + + + + + + + + + + + + + + diff --git a/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs b/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs new file mode 100644 index 0000000000..a1eeae70de --- /dev/null +++ b/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs @@ -0,0 +1,32 @@ +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.CommandLine; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.RunTests; +using Microsoft.ML.Runtime.Sweeper; +using System; +using System.IO; +using Xunit; + +namespace Microsoft.ML.Sweeper.Tests +{ + public class SweeperTest + { + [Fact] + public void SweeperReturnsDistinctValues() + { + var args = new DiscreteParamArguments(); + args.Name = "Amazing"; + args.Values = new string[] { "one" }; + var valueGenerator = new DiscreteValueGenerator(args); + using (var writer = new StreamWriter(new MemoryStream())) + using (var env = new TlcEnvironment(42, outWriter: writer, errWriter: writer)) + { + var sweeper = new UniformRandomSweeper(env, new SweeperBase.ArgumentsBase(), new[] { valueGenerator }); + var results = sweeper.ProposeSweeps(2); + Assert.NotNull(results); + int length = results.Length; + Assert.Equal(1, length); + } + } + } +} From 2bd9cfb8d428b27601d4433aba4e411c8b4ca159 Mon Sep 17 00:00:00 2001 From: Gabriel Nepomuceno Date: Fri, 15 Jun 2018 22:16:32 +0100 Subject: [PATCH 2/2] Correcting singleton and adding tests --- src/Microsoft.ML.Sweeper/Algorithms/Grid.cs | 10 ++-- .../Microsoft.ML.Sweeper.Tests.csproj | 1 + .../Microsoft.ML.Sweeper.Tests/SweeperTest.cs | 32 ----------- .../SweeperUniqueValuesTest.cs | 53 +++++++++++++++++++ 4 files changed, 59 insertions(+), 37 deletions(-) delete mode 100644 test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs create mode 100644 test/Microsoft.ML.Sweeper.Tests/SweeperUniqueValuesTest.cs diff --git a/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs b/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs index aeb66137fb..b0d4f385db 100644 --- a/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs +++ b/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs @@ -80,12 +80,11 @@ public virtual ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable previousRuns) @@ -150,7 +149,7 @@ public RandomGridSweeper(IHostEnvironment env, Arguments args, IValueGenerator[] } } - public override ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable previousRuns) + public override ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable previousRuns = null) { if (_nGridPoints == 0) return base.ProposeSweeps(maxSweeps, previousRuns); @@ -173,7 +172,8 @@ public override ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable + diff --git a/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs b/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs deleted file mode 100644 index a1eeae70de..0000000000 --- a/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs +++ /dev/null @@ -1,32 +0,0 @@ -using Microsoft.ML.Runtime; -using Microsoft.ML.Runtime.CommandLine; -using Microsoft.ML.Runtime.Data; -using Microsoft.ML.Runtime.RunTests; -using Microsoft.ML.Runtime.Sweeper; -using System; -using System.IO; -using Xunit; - -namespace Microsoft.ML.Sweeper.Tests -{ - public class SweeperTest - { - [Fact] - public void SweeperReturnsDistinctValues() - { - var args = new DiscreteParamArguments(); - args.Name = "Amazing"; - args.Values = new string[] { "one" }; - var valueGenerator = new DiscreteValueGenerator(args); - using (var writer = new StreamWriter(new MemoryStream())) - using (var env = new TlcEnvironment(42, outWriter: writer, errWriter: writer)) - { - var sweeper = new UniformRandomSweeper(env, new SweeperBase.ArgumentsBase(), new[] { valueGenerator }); - var results = sweeper.ProposeSweeps(2); - Assert.NotNull(results); - int length = results.Length; - Assert.Equal(1, length); - } - } - } -} diff --git a/test/Microsoft.ML.Sweeper.Tests/SweeperUniqueValuesTest.cs b/test/Microsoft.ML.Sweeper.Tests/SweeperUniqueValuesTest.cs new file mode 100644 index 0000000000..fecd0322ba --- /dev/null +++ b/test/Microsoft.ML.Sweeper.Tests/SweeperUniqueValuesTest.cs @@ -0,0 +1,53 @@ +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.CommandLine; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.PipelineInference; +using Microsoft.ML.Runtime.RunTests; +using Microsoft.ML.Runtime.Sweeper; +using System; +using System.IO; +using Xunit; + +namespace Microsoft.ML.Sweeper.Tests +{ + public class SweeperUniqueValuesTest + { + [Fact] + public void SweeperReturnsDistinctValuesForUniformRandomSweeper() + { + DiscreteValueGenerator valueGenerator = SetSingleParameter(); + using (var writer = new StreamWriter(new MemoryStream())) + using (var env = new TlcEnvironment(42, outWriter: writer, errWriter: writer)) + { + var sweeper = new UniformRandomSweeper(env, new SweeperBase.ArgumentsBase(), new[] { valueGenerator }); + var results = sweeper.ProposeSweeps(5000); + Assert.NotNull(results); + int length = results.Length; + Assert.Equal(2, length); + } + } + [Fact] + public void SweeperReturnsDistinctValuesForRandomGridSweeper() + { + DiscreteValueGenerator valueGenerator = SetSingleParameter(); + using (var writer = new StreamWriter(new MemoryStream())) + using (var env = new TlcEnvironment(42, outWriter: writer, errWriter: writer)) + { + var sweeper = new RandomGridSweeper(env, new RandomGridSweeper.Arguments(), new[] { valueGenerator }); + var results = sweeper.ProposeSweeps(5000); + Assert.NotNull(results); + int length = results.Length; + Assert.Equal(2, length); + } + } + + private static DiscreteValueGenerator SetSingleParameter() + { + var args = new DiscreteParamArguments(); + args.Name = "TestParam"; + args.Values = new string[] { "one", "two" }; + var valueGenerator = new DiscreteValueGenerator(args); + return valueGenerator; + } + } +}