From 750d0925ef2799d667123535deff14b88e0753ec Mon Sep 17 00:00:00 2001 From: hrntsm Date: Wed, 6 Jul 2022 23:08:58 +0900 Subject: [PATCH 01/11] Add timeout settings to optimize --- Tunny/Settings/Optimize.cs | 1 + Tunny/Solver/OptunaAlgorithm.cs | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Tunny/Settings/Optimize.cs b/Tunny/Settings/Optimize.cs index 99f7678b..2f0cc53e 100644 --- a/Tunny/Settings/Optimize.cs +++ b/Tunny/Settings/Optimize.cs @@ -6,5 +6,6 @@ public class Optimize public int NumberOfTrials { get; set; } = 100; public bool LoadExistStudy { get; set; } = true; public int SelectSampler { get; set; } + public double Timeout { get; set; } = double.MaxValue; } } diff --git a/Tunny/Solver/OptunaAlgorithm.cs b/Tunny/Solver/OptunaAlgorithm.cs index edc01878..4c887f7d 100644 --- a/Tunny/Solver/OptunaAlgorithm.cs +++ b/Tunny/Solver/OptunaAlgorithm.cs @@ -39,6 +39,7 @@ public void Solve() int variableCount = Lb.Length; int samplerType = Settings.Optimize.SelectSampler; int nTrials = Settings.Optimize.NumberOfTrials; + double timeout = Settings.Optimize.Timeout; int nObjective = ObjNickName.Length; string[] directions = new string[nObjective]; for (int i = 0; i < nObjective; i++) @@ -70,9 +71,17 @@ public void Solve() double[] xTest = new double[variableCount]; var result = new EvaluatedGHResult(); - for (int i = 0; i < nTrials; i++) + + int trialNum = 0; + DateTime startTime = DateTime.Now; + while (true) { - int progress = i * 100 / nTrials; + if (trialNum == nTrials || (DateTime.Now - startTime).TotalSeconds >= timeout) + { + break; + } + + int progress = trialNum * 100 / nTrials; dynamic trial = study.ask(); //TODO: Is this the correct way to handle the case of null? @@ -107,6 +116,7 @@ public void Solve() catch { } + trialNum++; } if (nObjective == 1) From e59d85a61cef5884ca1007335362b9b19aed6bf0 Mon Sep 17 00:00:00 2001 From: hrntsm Date: Wed, 6 Jul 2022 23:44:59 +0900 Subject: [PATCH 02/11] Add timeout UI --- Tunny/Settings/Optimize.cs | 2 +- Tunny/Solver/OptunaAlgorithm.cs | 2 +- Tunny/UI/OptimizationWindow.Designer.cs | 45 +++++++++++++++++++++-- Tunny/UI/OptimizationWindow.cs | 2 + Tunny/UI/OptimizationWindow.resx | 3 ++ Tunny/UI/OptimizeWindowTab/OptimizeTab.cs | 3 +- 6 files changed, 50 insertions(+), 7 deletions(-) diff --git a/Tunny/Settings/Optimize.cs b/Tunny/Settings/Optimize.cs index 2f0cc53e..a4d59f8f 100644 --- a/Tunny/Settings/Optimize.cs +++ b/Tunny/Settings/Optimize.cs @@ -6,6 +6,6 @@ public class Optimize public int NumberOfTrials { get; set; } = 100; public bool LoadExistStudy { get; set; } = true; public int SelectSampler { get; set; } - public double Timeout { get; set; } = double.MaxValue; + public double Timeout { get; set; } } } diff --git a/Tunny/Solver/OptunaAlgorithm.cs b/Tunny/Solver/OptunaAlgorithm.cs index 4c887f7d..2528626b 100644 --- a/Tunny/Solver/OptunaAlgorithm.cs +++ b/Tunny/Solver/OptunaAlgorithm.cs @@ -39,7 +39,7 @@ public void Solve() int variableCount = Lb.Length; int samplerType = Settings.Optimize.SelectSampler; int nTrials = Settings.Optimize.NumberOfTrials; - double timeout = Settings.Optimize.Timeout; + double timeout = Settings.Optimize.Timeout <= 0 ? double.MaxValue : Settings.Optimize.Timeout; int nObjective = ObjNickName.Length; string[] directions = new string[nObjective]; for (int i = 0; i < nObjective; i++) diff --git a/Tunny/UI/OptimizationWindow.Designer.cs b/Tunny/UI/OptimizationWindow.Designer.cs index bce846f9..0611b898 100644 --- a/Tunny/UI/OptimizationWindow.Designer.cs +++ b/Tunny/UI/OptimizationWindow.Designer.cs @@ -29,6 +29,7 @@ protected override void Dispose(bool disposing) /// private void InitializeComponent() { + this.components = new System.ComponentModel.Container(); System.ComponentModel.ComponentResourceManager resources = new System.ComponentModel.ComponentResourceManager(typeof(OptimizationWindow)); this.optimizeRunButton = new System.Windows.Forms.Button(); this.optimizeBackgroundWorker = new System.ComponentModel.BackgroundWorker(); @@ -68,6 +69,9 @@ private void InitializeComponent() this.openResultFolderButton = new System.Windows.Forms.Button(); this.clearResultButton = new System.Windows.Forms.Button(); this.outputResultBackgroundWorker = new System.ComponentModel.BackgroundWorker(); + this.Timeout = new System.Windows.Forms.Label(); + this.timeoutNumUpDown = new System.Windows.Forms.NumericUpDown(); + this.toolTip1 = new System.Windows.Forms.ToolTip(this.components); ((System.ComponentModel.ISupportInitialize)(this.nTrialNumUpDown)).BeginInit(); this.optimizeTabControl.SuspendLayout(); this.optimizeTabPage.SuspendLayout(); @@ -75,6 +79,7 @@ private void InitializeComponent() this.outputTabPage.SuspendLayout(); this.settingsTabPage.SuspendLayout(); this.fileTabPage.SuspendLayout(); + ((System.ComponentModel.ISupportInitialize)(this.timeoutNumUpDown)).BeginInit(); this.SuspendLayout(); // // optimizeRunButton @@ -134,7 +139,7 @@ private void InitializeComponent() this.loadIfExistsCheckBox.AutoSize = true; this.loadIfExistsCheckBox.Checked = true; this.loadIfExistsCheckBox.CheckState = System.Windows.Forms.CheckState.Checked; - this.loadIfExistsCheckBox.Location = new System.Drawing.Point(20, 135); + this.loadIfExistsCheckBox.Location = new System.Drawing.Point(19, 156); this.loadIfExistsCheckBox.Margin = new System.Windows.Forms.Padding(4, 6, 4, 6); this.loadIfExistsCheckBox.Name = "loadIfExistsCheckBox"; this.loadIfExistsCheckBox.Size = new System.Drawing.Size(237, 27); @@ -162,7 +167,7 @@ private void InitializeComponent() this.samplerComboBox.Location = new System.Drawing.Point(148, 12); this.samplerComboBox.Margin = new System.Windows.Forms.Padding(4, 6, 4, 6); this.samplerComboBox.Name = "samplerComboBox"; - this.samplerComboBox.Size = new System.Drawing.Size(208, 31); + this.samplerComboBox.Size = new System.Drawing.Size(214, 31); this.samplerComboBox.TabIndex = 7; // // samplerTypeText @@ -178,7 +183,7 @@ private void InitializeComponent() // studyNameLabel // this.studyNameLabel.AutoSize = true; - this.studyNameLabel.Location = new System.Drawing.Point(15, 172); + this.studyNameLabel.Location = new System.Drawing.Point(14, 193); this.studyNameLabel.Margin = new System.Windows.Forms.Padding(4, 0, 4, 0); this.studyNameLabel.Name = "studyNameLabel"; this.studyNameLabel.Size = new System.Drawing.Size(116, 23); @@ -187,7 +192,7 @@ private void InitializeComponent() // // studyNameTextBox // - this.studyNameTextBox.Location = new System.Drawing.Point(152, 166); + this.studyNameTextBox.Location = new System.Drawing.Point(151, 187); this.studyNameTextBox.Margin = new System.Windows.Forms.Padding(4, 6, 4, 6); this.studyNameTextBox.Name = "studyNameTextBox"; this.studyNameTextBox.Size = new System.Drawing.Size(205, 30); @@ -211,6 +216,8 @@ private void InitializeComponent() // // optimizeTabPage // + this.optimizeTabPage.Controls.Add(this.timeoutNumUpDown); + this.optimizeTabPage.Controls.Add(this.Timeout); this.optimizeTabPage.Controls.Add(this.studyNameTextBox); this.optimizeTabPage.Controls.Add(this.samplerComboBox); this.optimizeTabPage.Controls.Add(this.studyNameLabel); @@ -513,6 +520,32 @@ private void InitializeComponent() this.clearResultButton.Text = "Clear result flie"; this.clearResultButton.UseVisualStyleBackColor = true; this.clearResultButton.Click += new System.EventHandler(this.ClearResultButton_Click); + // + // Timeout + // + this.Timeout.AutoSize = true; + this.Timeout.Location = new System.Drawing.Point(16, 102); + this.Timeout.Margin = new System.Windows.Forms.Padding(4, 0, 4, 0); + this.Timeout.Name = "Timeout"; + this.Timeout.Size = new System.Drawing.Size(132, 23); + this.Timeout.TabIndex = 11; + this.Timeout.Text = "Timeout (sec)"; + // + // timeoutNumUpDown + // + this.timeoutNumUpDown.Location = new System.Drawing.Point(197, 102); + this.timeoutNumUpDown.Margin = new System.Windows.Forms.Padding(4, 6, 4, 6); + this.timeoutNumUpDown.Maximum = new decimal(new int[] { + 10000000, + 0, + 0, + 0}); + this.timeoutNumUpDown.Name = "timeoutNumUpDown"; + this.timeoutNumUpDown.Size = new System.Drawing.Size(165, 30); + this.timeoutNumUpDown.TabIndex = 12; + this.timeoutNumUpDown.ThousandsSeparator = true; + this.toolTip1.SetToolTip(this.timeoutNumUpDown, "After this time has elapsed from the start of optimization, optimization is stopp" + + "ed."); // // OptimizationWindow // @@ -537,6 +570,7 @@ private void InitializeComponent() this.outputTabPage.PerformLayout(); this.settingsTabPage.ResumeLayout(false); this.fileTabPage.ResumeLayout(false); + ((System.ComponentModel.ISupportInitialize)(this.timeoutNumUpDown)).EndInit(); this.ResumeLayout(false); } @@ -581,6 +615,9 @@ private void InitializeComponent() private System.Windows.Forms.TabPage fileTabPage; private System.Windows.Forms.Button openResultFolderButton; private System.Windows.Forms.Button clearResultButton; + private System.Windows.Forms.NumericUpDown timeoutNumUpDown; + private System.Windows.Forms.ToolTip toolTip1; + private System.Windows.Forms.Label Timeout; } } diff --git a/Tunny/UI/OptimizationWindow.cs b/Tunny/UI/OptimizationWindow.cs index 92fff058..eaed97a8 100644 --- a/Tunny/UI/OptimizationWindow.cs +++ b/Tunny/UI/OptimizationWindow.cs @@ -83,6 +83,7 @@ private void InitializeUIValues() { samplerComboBox.SelectedIndex = _settings.Optimize.SelectSampler; nTrialNumUpDown.Value = _settings.Optimize.NumberOfTrials; + timeoutNumUpDown.Value = (decimal)_settings.Optimize.Timeout; loadIfExistsCheckBox.Checked = _settings.Optimize.LoadExistStudy; studyNameTextBox.Text = _settings.StudyName; outputModelNumTextBox.Text = _settings.Result.OutputNumberString; @@ -127,6 +128,7 @@ private void SaveUIValues() { _settings.Optimize.SelectSampler = samplerComboBox.SelectedIndex; _settings.Optimize.NumberOfTrials = (int)nTrialNumUpDown.Value; + _settings.Optimize.Timeout = (double)timeoutNumUpDown.Value; _settings.Optimize.LoadExistStudy = loadIfExistsCheckBox.Checked; _settings.StudyName = studyNameTextBox.Text; _settings.Result.OutputNumberString = outputModelNumTextBox.Text; diff --git a/Tunny/UI/OptimizationWindow.resx b/Tunny/UI/OptimizationWindow.resx index c54977f1..1796dc55 100644 --- a/Tunny/UI/OptimizationWindow.resx +++ b/Tunny/UI/OptimizationWindow.resx @@ -120,6 +120,9 @@ 342, 17 + + 633, 17 + 17, 17 diff --git a/Tunny/UI/OptimizeWindowTab/OptimizeTab.cs b/Tunny/UI/OptimizeWindowTab/OptimizeTab.cs index c7733e8f..209b7ff9 100644 --- a/Tunny/UI/OptimizeWindowTab/OptimizeTab.cs +++ b/Tunny/UI/OptimizeWindowTab/OptimizeTab.cs @@ -20,6 +20,7 @@ private void OptimizeRunButton_Click(object sender, EventArgs e) _settings.Optimize.NumberOfTrials = (int)nTrialNumUpDown.Value; _settings.Optimize.SelectSampler = samplerComboBox.SelectedIndex; _settings.StudyName = studyNameTextBox.Text; + _settings.Optimize.Timeout = (double)timeoutNumUpDown.Value; OptimizeLoop.Settings = _settings; if (!CheckInputValue(ghCanvas)) @@ -64,7 +65,7 @@ private void OptimizeStopButton_Click(object sender, EventArgs e) if (optimizeBackgroundWorker != null) { - optimizeBackgroundWorker.CancelAsync(); + optimizeBackgroundWorker.Dispose(); } //Enable GUI From b994f19919d49b5b31d07bdec8dde690c286105c Mon Sep 17 00:00:00 2001 From: hrntsm Date: Wed, 6 Jul 2022 23:48:43 +0900 Subject: [PATCH 03/11] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa01958e..9d20e9ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Please see [here](https://github.com/hrntsm/Tunny/releases) for the data release - Components that output each attribute - Python package licenses to clearly state the license of each package. - requirements.txt file to avoid conflict python packages versions. +- Implemented Timeout to stop optimization over time. ### Changed From 245c543d3f2724b031f875283b39a96320352553 Mon Sep 17 00:00:00 2001 From: hrntsm Date: Thu, 7 Jul 2022 09:48:48 +0900 Subject: [PATCH 04/11] Clean sampler settings --- Tunny/Optimization/OptimizeLoop.cs | 4 +- Tunny/Optimization/OutputLoop.cs | 4 +- .../Algorithm.cs} | 117 ++++-------------- Tunny/Solver/Optuna/Sampler.cs | 80 ++++++++++++ Tunny/Solver/{Optuna.cs => Optuna/optuna.cs} | 8 +- Tunny/UI/OptimizeWindowTab/VisualizeTab.cs | 4 +- Tunny/Util/SliderValueParameters.cs | 10 ++ 7 files changed, 126 insertions(+), 101 deletions(-) rename Tunny/Solver/{OptunaAlgorithm.cs => Optuna/Algorithm.cs} (61%) create mode 100644 Tunny/Solver/Optuna/Sampler.cs rename Tunny/Solver/{Optuna.cs => Optuna/optuna.cs} (97%) create mode 100644 Tunny/Util/SliderValueParameters.cs diff --git a/Tunny/Optimization/OptimizeLoop.cs b/Tunny/Optimization/OptimizeLoop.cs index cf91ecfb..ab4e27c6 100644 --- a/Tunny/Optimization/OptimizeLoop.cs +++ b/Tunny/Optimization/OptimizeLoop.cs @@ -7,7 +7,7 @@ using Tunny.Component; using Tunny.Settings; -using Tunny.Solver; +using Tunny.Solver.Optuna; using Tunny.UI; using Tunny.Util; @@ -54,7 +54,7 @@ private static double[] RunOptimizationLoop(BackgroundWorker worker) return new[] { double.NaN }; } - var optunaSolver = new Optuna(s_component.GhInOut.ComponentFolder); + var optunaSolver = new optuna(s_component.GhInOut.ComponentFolder); bool solverStarted = optunaSolver.RunSolver( variables, objectives, EvaluateFunction, Settings); diff --git a/Tunny/Optimization/OutputLoop.cs b/Tunny/Optimization/OutputLoop.cs index 477cfda2..640887ac 100644 --- a/Tunny/Optimization/OutputLoop.cs +++ b/Tunny/Optimization/OutputLoop.cs @@ -6,7 +6,7 @@ using Rhino.Geometry; using Tunny.Component; -using Tunny.Solver; +using Tunny.Solver.Optuna; using Tunny.Type; using Tunny.UI; using Tunny.Util; @@ -29,7 +29,7 @@ internal static void Run(object sender, DoWorkEventArgs e) var fishes = new List(); - var optunaSolver = new Optuna(s_component.GhInOut.ComponentFolder); + var optunaSolver = new optuna(s_component.GhInOut.ComponentFolder); ModelResult[] modelResult = optunaSolver.GetModelResult(Indices, StudyName); if (modelResult.Length == 0) { diff --git a/Tunny/Solver/OptunaAlgorithm.cs b/Tunny/Solver/Optuna/Algorithm.cs similarity index 61% rename from Tunny/Solver/OptunaAlgorithm.cs rename to Tunny/Solver/Optuna/Algorithm.cs index 2528626b..25f016b5 100644 --- a/Tunny/Solver/OptunaAlgorithm.cs +++ b/Tunny/Solver/Optuna/Algorithm.cs @@ -7,36 +7,39 @@ using Tunny.Optimization; using Tunny.Settings; +using Tunny.Util; -namespace Tunny.Solver +namespace Tunny.Solver.Optuna { - public class OptunaAlgorithm + public class Algorithm { - private double[] Lb { get; set; } - private double[] Ub { get; set; } - private string[] VarNickName { get; set; } + private SliderValueParameters Sliders { get; set; } private string[] ObjNickName { get; set; } private TunnySettings Settings { get; set; } private Func EvalFunc { get; set; } private double[] XOpt { get; set; } private double[] FxOpt { get; set; } - public OptunaAlgorithm( + public Algorithm( double[] lb, double[] ub, string[] varNickName, string[] objNickName, TunnySettings settings, Func evalFunc) { - Lb = lb; - Ub = ub; - VarNickName = varNickName; + Sliders = new SliderValueParameters + { + LowerBond = lb, + UpperBond = ub, + NickName = varNickName, + Count = lb.Length + }; ObjNickName = objNickName; Settings = settings; EvalFunc = evalFunc; + } public void Solve() { - int variableCount = Lb.Length; int samplerType = Settings.Optimize.SelectSampler; int nTrials = Settings.Optimize.NumberOfTrials; double timeout = Settings.Optimize.Timeout <= 0 ? double.MaxValue : Settings.Optimize.Timeout; @@ -51,7 +54,7 @@ public void Solve() using (Py.GIL()) { dynamic optuna = Py.Import("optuna"); - dynamic sampler = SetSamplerSettings(variableCount, samplerType, ref nTrials, optuna); + dynamic sampler = SetSamplerSettings(Sliders, samplerType, ref nTrials, optuna); dynamic study = optuna.create_study( sampler: sampler, @@ -69,7 +72,7 @@ public void Solve() name.Remove(name.Length - 1, 1); SetStudyUserAttr(study, name); - double[] xTest = new double[variableCount]; + double[] xTest = new double[Sliders.Count]; var result = new EvaluatedGHResult(); int trialNum = 0; @@ -91,9 +94,9 @@ public void Solve() int nullCount = 0; while (nullCount < 10) { - for (int j = 0; j < variableCount; j++) + for (int j = 0; j < Sliders.Count; j++) { - xTest[j] = trial.suggest_uniform(VarNickName[j], Lb[j], Ub[j]); + xTest[j] = trial.suggest_uniform(Sliders.NickName[j], Sliders.LowerBond[j], Sliders.UpperBond[j]); } result = EvalFunc(xTest, progress); @@ -123,13 +126,13 @@ public void Solve() { double[] values = (double[])study.best_params.values(); string[] keys = (string[])study.best_params.keys(); - double[] opt = new double[VarNickName.Length]; + double[] opt = new double[Sliders.Count]; - for (int i = 0; i < VarNickName.Length; i++) + for (int i = 0; i < Sliders.Count; i++) { for (int j = 0; j < keys.Length; j++) { - if (keys[j] == VarNickName[i]) + if (keys[j] == Sliders.NickName[i]) { opt[i] = values[j]; } @@ -179,25 +182,25 @@ private static void SetTrialUserAttr(EvaluatedGHResult result, dynamic trial) } } - private dynamic SetSamplerSettings(int n, int samplerType, ref int nTrials, dynamic optuna) + private dynamic SetSamplerSettings(SliderValueParameters sliders, int samplerType, ref int nTrials, dynamic optuna) { dynamic sampler; switch (samplerType) { case 0: - sampler = SetTPESamplerSettings(optuna); + sampler = Sampler.TPE(optuna, Settings); break; case 1: - sampler = SetNSGAIISamplerSettings(optuna); + sampler = Sampler.NSGAII(optuna, Settings); break; case 2: - sampler = SetCmaEsSamplerSettings(optuna); + sampler = Sampler.CmaEs(optuna, Settings); break; case 3: - sampler = SetRandomSamplerSettings(optuna); + sampler = Sampler.Random(optuna, Settings); break; case 4: - sampler = SetGridSamplerSettings(n, ref nTrials, optuna); + sampler = Sampler.Grid(optuna, sliders, ref nTrials); break; default: throw new ArgumentException("Unknown sampler type"); @@ -205,74 +208,6 @@ private dynamic SetSamplerSettings(int n, int samplerType, ref int nTrials, dyna return sampler; } - private dynamic SetRandomSamplerSettings(dynamic optuna) - { - Settings.Random random = Settings.Optimize.Sampler.Random; - return optuna.samplers.RandomSampler( - seed: random.Seed - ); - } - - private dynamic SetCmaEsSamplerSettings(dynamic optuna) - { - CmaEs cmaEs = Settings.Optimize.Sampler.CmaEs; - return optuna.samplers.CmaEsSampler( - sigma0: cmaEs.Sigma0, - n_startup_trials: cmaEs.NStartupTrials, - warn_independent_sampling: cmaEs.WarnIndependentSampling, - seed: cmaEs.Seed, - consider_pruned_trials: cmaEs.ConsiderPrunedTrials, - restart_strategy: cmaEs.RestartStrategy, - inc_popsize: cmaEs.IncPopsize, - use_separable_cma: cmaEs.UseSeparableCma - ); - } - - private dynamic SetGridSamplerSettings(int n, ref int nTrials, dynamic optuna) - { - var searchSpace = new Dictionary>(); - for (int i = 0; i < n; i++) - { - var numSpace = new List(); - for (int j = 0; j < nTrials; j++) - { - numSpace.Add(Lb[i] + ((Ub[i] - Lb[i]) * j / (nTrials - 1))); - } - searchSpace.Add(VarNickName[i], numSpace); - } - nTrials = (int)Math.Pow(nTrials, n); - return optuna.samplers.GridSampler(searchSpace); - } - - private dynamic SetNSGAIISamplerSettings(dynamic optuna) - { - NSGAII nsga2 = Settings.Optimize.Sampler.NsgaII; - return optuna.samplers.NSGAIISampler( - population_size: nsga2.PopulationSize, - mutation_prob: nsga2.MutationProb, - crossover_prob: nsga2.CrossoverProb, - swapping_prob: nsga2.SwappingProb, - seed: nsga2.Seed - ); - } - - private dynamic SetTPESamplerSettings(dynamic optuna) - { - Tpe tpe = Settings.Optimize.Sampler.Tpe; - return optuna.samplers.TPESampler( - seed: tpe.Seed, - consider_prior: tpe.ConsiderPrior, - prior_weight: 1.0, - consider_magic_clip: tpe.ConsiderMagicClip, - consider_endpoints: tpe.ConsiderEndpoints, - n_startup_trials: tpe.NStartupTrials, - n_ei_candidates: tpe.NEICandidates, - multivariate: tpe.Multivariate, - group: tpe.Group, - warn_independent_sampling: tpe.WarnIndependentSampling, - constant_liar: tpe.ConstantLiar - ); - } public double[] GetXOptimum() { diff --git a/Tunny/Solver/Optuna/Sampler.cs b/Tunny/Solver/Optuna/Sampler.cs new file mode 100644 index 00000000..ab6d3722 --- /dev/null +++ b/Tunny/Solver/Optuna/Sampler.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; + +using Tunny.Settings; +using Tunny.Util; + +namespace Tunny.Solver.Optuna +{ + public class Sampler + { + internal static dynamic Random(dynamic optuna, TunnySettings settings) + { + Settings.Random random = settings.Optimize.Sampler.Random; + return optuna.samplers.RandomSampler( + seed: random.Seed + ); + } + + internal static dynamic CmaEs(dynamic optuna, TunnySettings settings) + { + CmaEs cmaEs = settings.Optimize.Sampler.CmaEs; + return optuna.samplers.CmaEsSampler( + sigma0: cmaEs.Sigma0, + n_startup_trials: cmaEs.NStartupTrials, + warn_independent_sampling: cmaEs.WarnIndependentSampling, + seed: cmaEs.Seed, + consider_pruned_trials: cmaEs.ConsiderPrunedTrials, + restart_strategy: cmaEs.RestartStrategy, + inc_popsize: cmaEs.IncPopsize, + use_separable_cma: cmaEs.UseSeparableCma + ); + } + + internal static dynamic Grid(dynamic optuna, SliderValueParameters sliders, ref int nTrials) + { + var searchSpace = new Dictionary>(); + for (int i = 0; i < sliders.LowerBond.Length; i++) + { + var numSpace = new List(); + for (int j = 0; j < nTrials; j++) + { + numSpace.Add(sliders.LowerBond[i] + (sliders.UpperBond[i] - sliders.LowerBond[i]) * j / (nTrials - 1)); + } + searchSpace.Add(sliders.NickName[i], numSpace); + } + nTrials = (int)Math.Pow(nTrials, sliders.Count); + return optuna.samplers.GridSampler(searchSpace); + } + + internal static dynamic NSGAII(dynamic optuna, TunnySettings settings) + { + NSGAII nsga2 = settings.Optimize.Sampler.NsgaII; + return optuna.samplers.NSGAIISampler( + population_size: nsga2.PopulationSize, + mutation_prob: nsga2.MutationProb, + crossover_prob: nsga2.CrossoverProb, + swapping_prob: nsga2.SwappingProb, + seed: nsga2.Seed + ); + } + + internal static dynamic TPE(dynamic optuna, TunnySettings settings) + { + Tpe tpe = settings.Optimize.Sampler.Tpe; + return optuna.samplers.TPESampler( + seed: tpe.Seed, + consider_prior: tpe.ConsiderPrior, + prior_weight: 1.0, + consider_magic_clip: tpe.ConsiderMagicClip, + consider_endpoints: tpe.ConsiderEndpoints, + n_startup_trials: tpe.NStartupTrials, + n_ei_candidates: tpe.NEICandidates, + multivariate: tpe.Multivariate, + group: tpe.Group, + warn_independent_sampling: tpe.WarnIndependentSampling, + constant_liar: tpe.ConstantLiar + ); + } + } +} diff --git a/Tunny/Solver/Optuna.cs b/Tunny/Solver/Optuna/optuna.cs similarity index 97% rename from Tunny/Solver/Optuna.cs rename to Tunny/Solver/Optuna/optuna.cs index cf475fbf..8fcb0e3b 100644 --- a/Tunny/Solver/Optuna.cs +++ b/Tunny/Solver/Optuna/optuna.cs @@ -12,16 +12,16 @@ using Tunny.UI; using Tunny.Util; -namespace Tunny.Solver +namespace Tunny.Solver.Optuna { - public class Optuna + public class optuna { public double[] XOpt { get; private set; } private double[] FxOpt { get; set; } private readonly string _componentFolder; - public Optuna(string componentFolder) + public optuna(string componentFolder) { _componentFolder = componentFolder; string envPath = PythonInstaller.GetEmbeddedPythonPath() + @"\python310.dll"; @@ -57,7 +57,7 @@ EvaluatedGHResult Eval(double[] x, int progress) try { - var tpe = new OptunaAlgorithm(lb, ub, varNickName, objNickName, settings, Eval); + var tpe = new Algorithm(lb, ub, varNickName, objNickName, settings, Eval); tpe.Solve(); XOpt = tpe.GetXOptimum(); FxOpt = tpe.GetFxOptimum(); diff --git a/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs b/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs index d11ecc43..c4becda5 100644 --- a/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs +++ b/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs @@ -2,7 +2,7 @@ using System.Diagnostics; using System.Windows.Forms; -using Tunny.Solver; +using Tunny.Solver.Optuna; using Tunny.Util; namespace Tunny.UI @@ -26,7 +26,7 @@ private void DashboardButton_Click(object sender, EventArgs e) private void SelectedTypePlotButton_Click(object sender, EventArgs e) { - var optuna = new Optuna(_component.GhInOut.ComponentFolder); + var optuna = new optuna(_component.GhInOut.ComponentFolder); optuna.ShowSelectedTypePlot(visualizeTypeComboBox.Text, studyNameTextBox.Text); } } diff --git a/Tunny/Util/SliderValueParameters.cs b/Tunny/Util/SliderValueParameters.cs new file mode 100644 index 00000000..0ced736a --- /dev/null +++ b/Tunny/Util/SliderValueParameters.cs @@ -0,0 +1,10 @@ +namespace Tunny.Util +{ + public class SliderValueParameters + { + public int Count { get; set; } + public double[] LowerBond { get; set; } + public double[] UpperBond { get; set; } + public string[] NickName { get; set; } + } +} From 27cb1a167add210bbf7ef4d42c58e1a817f8455b Mon Sep 17 00:00:00 2001 From: hrntsm Date: Thu, 7 Jul 2022 15:45:24 +0900 Subject: [PATCH 05/11] Clean sliderValueParam to variable --- Tunny/Solver/Optuna/Algorithm.cs | 30 ++++++++++++----------------- Tunny/Solver/Optuna/Sampler.cs | 10 +++++----- Tunny/Solver/Optuna/optuna.cs | 17 +++------------- Tunny/Util/GrasshopperInOut.cs | 4 ++-- Tunny/Util/SliderValueParameters.cs | 10 ---------- Tunny/Util/Variables.cs | 6 +++--- 6 files changed, 25 insertions(+), 52 deletions(-) delete mode 100644 Tunny/Util/SliderValueParameters.cs diff --git a/Tunny/Solver/Optuna/Algorithm.cs b/Tunny/Solver/Optuna/Algorithm.cs index 25f016b5..31cd9c56 100644 --- a/Tunny/Solver/Optuna/Algorithm.cs +++ b/Tunny/Solver/Optuna/Algorithm.cs @@ -13,7 +13,7 @@ namespace Tunny.Solver.Optuna { public class Algorithm { - private SliderValueParameters Sliders { get; set; } + private List Variables { get; set; } private string[] ObjNickName { get; set; } private TunnySettings Settings { get; set; } private Func EvalFunc { get; set; } @@ -21,17 +21,11 @@ public class Algorithm private double[] FxOpt { get; set; } public Algorithm( - double[] lb, double[] ub, string[] varNickName, string[] objNickName, + List variables, string[] objNickName, TunnySettings settings, Func evalFunc) { - Sliders = new SliderValueParameters - { - LowerBond = lb, - UpperBond = ub, - NickName = varNickName, - Count = lb.Length - }; + Variables = variables; ObjNickName = objNickName; Settings = settings; EvalFunc = evalFunc; @@ -54,7 +48,7 @@ public void Solve() using (Py.GIL()) { dynamic optuna = Py.Import("optuna"); - dynamic sampler = SetSamplerSettings(Sliders, samplerType, ref nTrials, optuna); + dynamic sampler = SetSamplerSettings(samplerType, ref nTrials, optuna); dynamic study = optuna.create_study( sampler: sampler, @@ -72,7 +66,7 @@ public void Solve() name.Remove(name.Length - 1, 1); SetStudyUserAttr(study, name); - double[] xTest = new double[Sliders.Count]; + double[] xTest = new double[Variables.Count]; var result = new EvaluatedGHResult(); int trialNum = 0; @@ -94,9 +88,9 @@ public void Solve() int nullCount = 0; while (nullCount < 10) { - for (int j = 0; j < Sliders.Count; j++) + for (int j = 0; j < Variables.Count; j++) { - xTest[j] = trial.suggest_uniform(Sliders.NickName[j], Sliders.LowerBond[j], Sliders.UpperBond[j]); + xTest[j] = trial.suggest_uniform(Variables[j].NickName, Variables[j].LowerBond, Variables[j].UpperBond); } result = EvalFunc(xTest, progress); @@ -126,13 +120,13 @@ public void Solve() { double[] values = (double[])study.best_params.values(); string[] keys = (string[])study.best_params.keys(); - double[] opt = new double[Sliders.Count]; + double[] opt = new double[Variables.Count]; - for (int i = 0; i < Sliders.Count; i++) + for (int i = 0; i < Variables.Count; i++) { for (int j = 0; j < keys.Length; j++) { - if (keys[j] == Sliders.NickName[i]) + if (keys[j] == Variables[i].NickName) { opt[i] = values[j]; } @@ -182,7 +176,7 @@ private static void SetTrialUserAttr(EvaluatedGHResult result, dynamic trial) } } - private dynamic SetSamplerSettings(SliderValueParameters sliders, int samplerType, ref int nTrials, dynamic optuna) + private dynamic SetSamplerSettings(int samplerType, ref int nTrials, dynamic optuna) { dynamic sampler; switch (samplerType) @@ -200,7 +194,7 @@ private dynamic SetSamplerSettings(SliderValueParameters sliders, int samplerTyp sampler = Sampler.Random(optuna, Settings); break; case 4: - sampler = Sampler.Grid(optuna, sliders, ref nTrials); + sampler = Sampler.Grid(optuna, Variables, ref nTrials); break; default: throw new ArgumentException("Unknown sampler type"); diff --git a/Tunny/Solver/Optuna/Sampler.cs b/Tunny/Solver/Optuna/Sampler.cs index ab6d3722..fb981f98 100644 --- a/Tunny/Solver/Optuna/Sampler.cs +++ b/Tunny/Solver/Optuna/Sampler.cs @@ -31,19 +31,19 @@ internal static dynamic CmaEs(dynamic optuna, TunnySettings settings) ); } - internal static dynamic Grid(dynamic optuna, SliderValueParameters sliders, ref int nTrials) + internal static dynamic Grid(dynamic optuna, List variables, ref int nTrials) { var searchSpace = new Dictionary>(); - for (int i = 0; i < sliders.LowerBond.Length; i++) + for (int i = 0; i < variables.Count; i++) { var numSpace = new List(); for (int j = 0; j < nTrials; j++) { - numSpace.Add(sliders.LowerBond[i] + (sliders.UpperBond[i] - sliders.LowerBond[i]) * j / (nTrials - 1)); + numSpace.Add(variables[i].LowerBond + (variables[i].UpperBond - variables[i].LowerBond) * j / (nTrials - 1)); } - searchSpace.Add(sliders.NickName[i], numSpace); + searchSpace.Add(variables[i].NickName, numSpace); } - nTrials = (int)Math.Pow(nTrials, sliders.Count); + nTrials = (int)Math.Pow(nTrials, variables.Count); return optuna.samplers.GridSampler(searchSpace); } diff --git a/Tunny/Solver/Optuna/optuna.cs b/Tunny/Solver/Optuna/optuna.cs index 8fcb0e3b..428704ae 100644 --- a/Tunny/Solver/Optuna/optuna.cs +++ b/Tunny/Solver/Optuna/optuna.cs @@ -34,21 +34,8 @@ public bool RunSolver( Func, int, EvaluatedGHResult> evaluate, TunnySettings settings) { - int dVar = variables.Count; - double[] lb = new double[dVar]; - double[] ub = new double[dVar]; - bool[] isInteger = new bool[dVar]; - string[] varNickName = new string[dVar]; string[] objNickName = objectives.Select(x => x.NickName).ToArray(); - for (int i = 0; i < dVar; i++) - { - lb[i] = Convert.ToDouble(variables[i].LowerBond); - ub[i] = Convert.ToDouble(variables[i].UpperBond); - isInteger[i] = variables[i].IsInteger; - varNickName[i] = variables[i].NickName; - } - EvaluatedGHResult Eval(double[] x, int progress) { var decimals = x.Select(Convert.ToDecimal).ToList(); @@ -57,7 +44,7 @@ EvaluatedGHResult Eval(double[] x, int progress) try { - var tpe = new Algorithm(lb, ub, varNickName, objNickName, settings, Eval); + var tpe = new Algorithm(variables, objNickName, settings, Eval); tpe.Solve(); XOpt = tpe.GetXOptimum(); FxOpt = tpe.GetFxOptimum(); @@ -79,6 +66,7 @@ EvaluatedGHResult Eval(double[] x, int progress) public void ShowSelectedTypePlot(string visualize, string studyName) { + //TODO: Use settings path to get the path of the database. string storage = "sqlite:///" + _componentFolder + "/Tunny_Opt_Result.db"; PythonEngine.Initialize(); using (Py.GIL()) @@ -148,6 +136,7 @@ public void ShowSelectedTypePlot(string visualize, string studyName) public ModelResult[] GetModelResult(int[] resultNum, string studyName) { + //TODO: Use settings path to get the path of the database. string storage = "sqlite:///" + _componentFolder + "/Tunny_Opt_Result.db"; var modelResult = new List(); PythonEngine.Initialize(); diff --git a/Tunny/Util/GrasshopperInOut.cs b/Tunny/Util/GrasshopperInOut.cs index a017fb4d..67e0dab5 100644 --- a/Tunny/Util/GrasshopperInOut.cs +++ b/Tunny/Util/GrasshopperInOut.cs @@ -127,7 +127,7 @@ private void SetInputSliderValues(ICollection variables) break; } - variables.Add(new Variable(lowerBond, upperBond, isInteger, nickName)); + variables.Add(new Variable(Convert.ToDouble(lowerBond), Convert.ToDouble(upperBond), isInteger, nickName)); } } @@ -144,7 +144,7 @@ private void SetInputGenePoolValues(ICollection variables) for (int j = 0; j < genePool.Count; j++) { string nickName = "genepool" + count++; - variables.Add(new Variable(lowerBond, upperBond, isInteger, nickName)); + variables.Add(new Variable(Convert.ToDouble(lowerBond), Convert.ToDouble(upperBond), isInteger, nickName)); } } } diff --git a/Tunny/Util/SliderValueParameters.cs b/Tunny/Util/SliderValueParameters.cs deleted file mode 100644 index 0ced736a..00000000 --- a/Tunny/Util/SliderValueParameters.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Tunny.Util -{ - public class SliderValueParameters - { - public int Count { get; set; } - public double[] LowerBond { get; set; } - public double[] UpperBond { get; set; } - public string[] NickName { get; set; } - } -} diff --git a/Tunny/Util/Variables.cs b/Tunny/Util/Variables.cs index 20dc5f6c..4407b955 100644 --- a/Tunny/Util/Variables.cs +++ b/Tunny/Util/Variables.cs @@ -2,12 +2,12 @@ namespace Tunny.Util { public struct Variable { - public decimal LowerBond { get; } - public decimal UpperBond { get; } + public double LowerBond { get; } + public double UpperBond { get; } public bool IsInteger { get; } public string NickName { get; } - public Variable(decimal lowerBond, decimal upperBond, bool isInteger, string nickName) + public Variable(double lowerBond, double upperBond, bool isInteger, string nickName) { LowerBond = lowerBond; UpperBond = upperBond; From 7c1e18db73a8e225221cce76f39292ad2e90d6c1 Mon Sep 17 00:00:00 2001 From: hrntsm Date: Thu, 7 Jul 2022 15:58:49 +0900 Subject: [PATCH 06/11] Fix visalize do not use setting storage path --- CHANGELOG.md | 2 +- Tunny/Optimization/OptimizeLoop.cs | 5 ++--- Tunny/Optimization/OutputLoop.cs | 4 +++- Tunny/Solver/Optuna/optuna.cs | 17 ++++++++--------- Tunny/UI/OptimizeWindowTab/OutputTab.cs | 1 + Tunny/UI/OptimizeWindowTab/VisualizeTab.cs | 2 +- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d20e9ca..17f8947e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ Please see [here](https://github.com/hrntsm/Tunny/releases) for the data release - Stopped sampling when there was no geometry input - Once optimize output error, the component won't run again - I've tried to do a proper Dispose to fix this problem, but it still doesn't work sometimes. -- Optuna-DashBoard and Delete Result Files functions do not work properly when a different Storage path is specified in Settings than the default. +- Optuna-DashBoard and storage relate functions do not work properly when a different Storage path is specified in Settings than the default. ## [0.3.0] -2022-05-03 diff --git a/Tunny/Optimization/OptimizeLoop.cs b/Tunny/Optimization/OptimizeLoop.cs index ab4e27c6..79815d69 100644 --- a/Tunny/Optimization/OptimizeLoop.cs +++ b/Tunny/Optimization/OptimizeLoop.cs @@ -54,10 +54,9 @@ private static double[] RunOptimizationLoop(BackgroundWorker worker) return new[] { double.NaN }; } - var optunaSolver = new optuna(s_component.GhInOut.ComponentFolder); + var optunaSolver = new Optuna(s_component.GhInOut.ComponentFolder, Settings); - bool solverStarted = optunaSolver.RunSolver( - variables, objectives, EvaluateFunction, Settings); + bool solverStarted = optunaSolver.RunSolver(variables, objectives, EvaluateFunction); return solverStarted ? optunaSolver.XOpt : new[] { double.NaN }; } diff --git a/Tunny/Optimization/OutputLoop.cs b/Tunny/Optimization/OutputLoop.cs index 640887ac..abfa4990 100644 --- a/Tunny/Optimization/OutputLoop.cs +++ b/Tunny/Optimization/OutputLoop.cs @@ -6,6 +6,7 @@ using Rhino.Geometry; using Tunny.Component; +using Tunny.Settings; using Tunny.Solver.Optuna; using Tunny.Type; using Tunny.UI; @@ -17,6 +18,7 @@ internal static class OutputLoop { private static BackgroundWorker s_worker; private static TunnyComponent s_component; + public static TunnySettings Settings; public static string StudyName; public static string[] NickNames; public static int[] Indices; @@ -29,7 +31,7 @@ internal static void Run(object sender, DoWorkEventArgs e) var fishes = new List(); - var optunaSolver = new optuna(s_component.GhInOut.ComponentFolder); + var optunaSolver = new Optuna(s_component.GhInOut.ComponentFolder, Settings); ModelResult[] modelResult = optunaSolver.GetModelResult(Indices, StudyName); if (modelResult.Length == 0) { diff --git a/Tunny/Solver/Optuna/optuna.cs b/Tunny/Solver/Optuna/optuna.cs index 428704ae..692d62fa 100644 --- a/Tunny/Solver/Optuna/optuna.cs +++ b/Tunny/Solver/Optuna/optuna.cs @@ -14,16 +14,18 @@ namespace Tunny.Solver.Optuna { - public class optuna + public class Optuna { public double[] XOpt { get; private set; } private double[] FxOpt { get; set; } private readonly string _componentFolder; + private readonly TunnySettings _settings; - public optuna(string componentFolder) + public Optuna(string componentFolder, TunnySettings settings) { _componentFolder = componentFolder; + _settings = settings; string envPath = PythonInstaller.GetEmbeddedPythonPath() + @"\python310.dll"; Environment.SetEnvironmentVariable("PYTHONNET_PYDLL", envPath, EnvironmentVariableTarget.Process); } @@ -31,8 +33,7 @@ public optuna(string componentFolder) public bool RunSolver( List variables, IEnumerable objectives, - Func, int, EvaluatedGHResult> evaluate, - TunnySettings settings) + Func, int, EvaluatedGHResult> evaluate) { string[] objNickName = objectives.Select(x => x.NickName).ToArray(); @@ -44,7 +45,7 @@ EvaluatedGHResult Eval(double[] x, int progress) try { - var tpe = new Algorithm(variables, objNickName, settings, Eval); + var tpe = new Algorithm(variables, objNickName, _settings, Eval); tpe.Solve(); XOpt = tpe.GetXOptimum(); FxOpt = tpe.GetFxOptimum(); @@ -66,8 +67,7 @@ EvaluatedGHResult Eval(double[] x, int progress) public void ShowSelectedTypePlot(string visualize, string studyName) { - //TODO: Use settings path to get the path of the database. - string storage = "sqlite:///" + _componentFolder + "/Tunny_Opt_Result.db"; + string storage = "sqlite:///" + _settings.Storage; PythonEngine.Initialize(); using (Py.GIL()) { @@ -136,8 +136,7 @@ public void ShowSelectedTypePlot(string visualize, string studyName) public ModelResult[] GetModelResult(int[] resultNum, string studyName) { - //TODO: Use settings path to get the path of the database. - string storage = "sqlite:///" + _componentFolder + "/Tunny_Opt_Result.db"; + string storage = "sqlite:///" + _settings.Storage; var modelResult = new List(); PythonEngine.Initialize(); using (Py.GIL()) diff --git a/Tunny/UI/OptimizeWindowTab/OutputTab.cs b/Tunny/UI/OptimizeWindowTab/OutputTab.cs index a451e20c..cae36f03 100644 --- a/Tunny/UI/OptimizeWindowTab/OutputTab.cs +++ b/Tunny/UI/OptimizeWindowTab/OutputTab.cs @@ -33,6 +33,7 @@ private void ReflectToSliderButton_Click(object sender, EventArgs e) private void RunOutputLoop(OutputMode mode) { OutputLoop.Mode = mode; + OutputLoop.Settings = _settings; OutputLoop.StudyName = studyNameTextBox.Text; OutputLoop.NickNames = _component.GhInOut.Variables.Select(x => x.NickName).ToArray(); int[] indices = outputModelNumTextBox.Text.Split(',').Select(int.Parse).ToArray(); diff --git a/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs b/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs index c4becda5..4ce1f948 100644 --- a/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs +++ b/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs @@ -26,7 +26,7 @@ private void DashboardButton_Click(object sender, EventArgs e) private void SelectedTypePlotButton_Click(object sender, EventArgs e) { - var optuna = new optuna(_component.GhInOut.ComponentFolder); + var optuna = new Optuna(_component.GhInOut.ComponentFolder, _settings); optuna.ShowSelectedTypePlot(visualizeTypeComboBox.Text, studyNameTextBox.Text); } } From c7a7cce64adc39e7daa8d3b3e6e5eb9f63522639 Mon Sep 17 00:00:00 2001 From: hrntsm Date: Thu, 7 Jul 2022 20:32:34 +0900 Subject: [PATCH 07/11] Fix CodeClimate issue --- Tunny/Solver/Optuna/Algorithm.cs | 118 +++++++++++----------- Tunny/Solver/Optuna/optuna.cs | 124 ++++++++++++------------ Tunny/UI/OptimizeWindowTab/OutputTab.cs | 7 +- 3 files changed, 131 insertions(+), 118 deletions(-) diff --git a/Tunny/Solver/Optuna/Algorithm.cs b/Tunny/Solver/Optuna/Algorithm.cs index 31cd9c56..6077d5d8 100644 --- a/Tunny/Solver/Optuna/Algorithm.cs +++ b/Tunny/Solver/Optuna/Algorithm.cs @@ -66,82 +66,87 @@ public void Solve() name.Remove(name.Length - 1, 1); SetStudyUserAttr(study, name); - double[] xTest = new double[Variables.Count]; - var result = new EvaluatedGHResult(); + RunOptimize(nTrials, timeout, study, out double[] xTest, out EvaluatedGHResult result); + SetResultValues(nObjective, study, xTest, result); + } + PythonEngine.Shutdown(); + } - int trialNum = 0; - DateTime startTime = DateTime.Now; - while (true) + private void SetResultValues(int nObjective, dynamic study, double[] xTest, EvaluatedGHResult result) + { + if (nObjective == 1) + { + double[] values = (double[])study.best_params.values(); + string[] keys = (string[])study.best_params.keys(); + double[] opt = new double[Variables.Count]; + + for (int i = 0; i < Variables.Count; i++) { - if (trialNum == nTrials || (DateTime.Now - startTime).TotalSeconds >= timeout) + for (int j = 0; j < keys.Length; j++) { - break; + if (keys[j] == Variables[i].NickName) + { + opt[i] = values[j]; + } } + } + XOpt = opt; + FxOpt = new[] { (double)study.best_value }; + } + else + { + XOpt = xTest; + FxOpt = result.ObjectiveValues.ToArray(); + } + } + + private void RunOptimize(int nTrials, double timeout, dynamic study, out double[] xTest, out EvaluatedGHResult result) + { + xTest = new double[Variables.Count]; + result = new EvaluatedGHResult(); + int trialNum = 0; + DateTime startTime = DateTime.Now; + while (true) + { + if (trialNum == nTrials || (DateTime.Now - startTime).TotalSeconds >= timeout) + { + break; + } - int progress = trialNum * 100 / nTrials; - dynamic trial = study.ask(); + int progress = trialNum * 100 / nTrials; + dynamic trial = study.ask(); - //TODO: Is this the correct way to handle the case of null? - //Other than TPE, the value is returned at random when retrying, so the value will be anything but null. - //TPEs, on the other hand, search for a specific location determined by GP, - //so the value tends to remain the same even after retries and there is no way to get out. - int nullCount = 0; - while (nullCount < 10) + //TODO: Is this the correct way to handle the case of null? + int nullCount = 0; + while (nullCount < 10) + { + for (int j = 0; j < Variables.Count; j++) { - for (int j = 0; j < Variables.Count; j++) - { - xTest[j] = trial.suggest_uniform(Variables[j].NickName, Variables[j].LowerBond, Variables[j].UpperBond); - } - result = EvalFunc(xTest, progress); - - if (result.ObjectiveValues.Contains(double.NaN)) - { - trial = study.ask(); - nullCount++; - } - else - { - break; - } + xTest[j] = trial.suggest_uniform(Variables[j].NickName, Variables[j].LowerBond, Variables[j].UpperBond); } + result = EvalFunc(xTest, progress); - SetTrialUserAttr(result, trial); - try + if (result.ObjectiveValues.Contains(double.NaN)) { - study.tell(trial, result.ObjectiveValues.ToArray()); + trial = study.ask(); + nullCount++; } - catch + else { + break; } - trialNum++; } - if (nObjective == 1) + SetTrialUserAttr(result, trial); + try { - double[] values = (double[])study.best_params.values(); - string[] keys = (string[])study.best_params.keys(); - double[] opt = new double[Variables.Count]; - - for (int i = 0; i < Variables.Count; i++) - { - for (int j = 0; j < keys.Length; j++) - { - if (keys[j] == Variables[i].NickName) - { - opt[i] = values[j]; - } - } - } - XOpt = opt; - FxOpt = new[] { (double)study.best_value }; + study.tell(trial, result.ObjectiveValues.ToArray()); } - else + catch { - XOpt = xTest; - FxOpt = result.ObjectiveValues.ToArray(); } + trialNum++; } - PythonEngine.Shutdown(); } private static void SetStudyUserAttr(dynamic study, StringBuilder name) @@ -202,7 +207,6 @@ private dynamic SetSamplerSettings(int samplerType, ref int nTrials, dynamic opt return sampler; } - public double[] GetXOptimum() { return XOpt; diff --git a/Tunny/Solver/Optuna/optuna.cs b/Tunny/Solver/Optuna/optuna.cs index 692d62fa..7704c2b7 100644 --- a/Tunny/Solver/Optuna/optuna.cs +++ b/Tunny/Solver/Optuna/optuna.cs @@ -17,8 +17,6 @@ namespace Tunny.Solver.Optuna public class Optuna { public double[] XOpt { get; private set; } - private double[] FxOpt { get; set; } - private readonly string _componentFolder; private readonly TunnySettings _settings; @@ -48,23 +46,26 @@ EvaluatedGHResult Eval(double[] x, int progress) var tpe = new Algorithm(variables, objNickName, _settings, Eval); tpe.Solve(); XOpt = tpe.GetXOptimum(); - FxOpt = tpe.GetFxOptimum(); TunnyMessageBox.Show("Solver completed successfully.", "Tunny"); - return true; } catch (Exception e) { - TunnyMessageBox.Show( - "Tunny runtime error:\n" + - "Please send below message (& gh file if possible) to Tunny support.\n\n" + - "\" " + e.Message + " \"", "Tunny", - MessageBoxButtons.OK, MessageBoxIcon.Error); + ShowErrorMessages(e); return false; } } + private static void ShowErrorMessages(Exception e) + { + TunnyMessageBox.Show( + "Tunny runtime error:\n" + + "Please send below message (& gh file if possible) to Tunny support.\n\n" + + "\" " + e.Message + " \"", "Tunny", + MessageBoxButtons.OK, MessageBoxIcon.Error); + } + public void ShowSelectedTypePlot(string visualize, string studyName) { string storage = "sqlite:///" + _settings.Storage; @@ -86,45 +87,7 @@ public void ShowSelectedTypePlot(string visualize, string studyName) string[] nickNames = ((string)study.user_attrs["objective_names"]).Split(','); try { - dynamic vis; - switch (visualize) - { - case "contour": - vis = optuna.visualization.plot_contour(study, target_name: nickNames[0]); - vis.show(); - break; - case "EDF": - vis = optuna.visualization.plot_edf(study, target_name: nickNames[0]); - vis.show(); - break; - case "intermediate values": - vis = optuna.visualization.plot_intermediate_values(study); - vis.show(); - break; - case "optimization history": - vis = optuna.visualization.plot_optimization_history(study, target_name: nickNames[0]); - vis.show(); - break; - case "parallel coordinate": - vis = optuna.visualization.plot_parallel_coordinate(study, target_name: nickNames[0]); - vis.show(); - break; - case "param importances": - vis = optuna.visualization.plot_param_importances(study, target_name: nickNames[0]); - vis.show(); - break; - case "pareto front": - vis = optuna.visualization.plot_pareto_front(study, target_names: nickNames); - vis.show(); - break; - case "slice": - vis = optuna.visualization.plot_slice(study, target_name: nickNames[0]); - vis.show(); - break; - default: - TunnyMessageBox.Show("This visualization type is not supported in this study case.", "Tunny"); - break; - } + ShowPlot(optuna, visualize, study, nickNames); } catch (Exception) { @@ -134,6 +97,42 @@ public void ShowSelectedTypePlot(string visualize, string studyName) PythonEngine.Shutdown(); } + private static void ShowPlot(dynamic optuna, string visualize, dynamic study, string[] nickNames) + { + dynamic vis; + switch (visualize) + { + case "contour": + vis = optuna.visualization.plot_contour(study, target_name: nickNames[0]); + break; + case "EDF": + vis = optuna.visualization.plot_edf(study, target_name: nickNames[0]); + break; + case "intermediate values": + vis = optuna.visualization.plot_intermediate_values(study); + break; + case "optimization history": + vis = optuna.visualization.plot_optimization_history(study, target_name: nickNames[0]); + break; + case "parallel coordinate": + vis = optuna.visualization.plot_parallel_coordinate(study, target_name: nickNames[0]); + break; + case "param importances": + vis = optuna.visualization.plot_param_importances(study, target_name: nickNames[0]); + break; + case "pareto front": + vis = optuna.visualization.plot_pareto_front(study, target_names: nickNames); + break; + case "slice": + vis = optuna.visualization.plot_slice(study, target_name: nickNames[0]); + break; + default: + TunnyMessageBox.Show("This visualization type is not supported in this study case.", "Tunny"); + return; + } + vis.show(); + } + public ModelResult[] GetModelResult(int[] resultNum, string studyName) { string storage = "sqlite:///" + _settings.Storage; @@ -154,34 +153,39 @@ public ModelResult[] GetModelResult(int[] resultNum, string studyName) return modelResult.ToArray(); } - if (resultNum[0] == -1) - { + SetTrialsToModelResult(resultNum, modelResult, study); + } + PythonEngine.Shutdown(); + + return modelResult.ToArray(); + } + + private void SetTrialsToModelResult(int[] resultNum, List modelResult, dynamic study) + { + switch (resultNum[0]) + { + case -1: var bestTrials = (dynamic[])study.best_trials; foreach (dynamic trial in bestTrials) { ParseTrial(modelResult, trial); } - } - else if (resultNum[0] == -10) - { + break; + case -10: var trials = (dynamic[])study.trials; foreach (dynamic trial in trials) { ParseTrial(modelResult, trial); } - } - else - { + break; + default: foreach (int res in resultNum) { dynamic trial = study.trials[res]; ParseTrial(modelResult, trial); } - } + break; } - PythonEngine.Shutdown(); - - return modelResult.ToArray(); } private static void ParseTrial(ICollection modelResult, dynamic trial) diff --git a/Tunny/UI/OptimizeWindowTab/OutputTab.cs b/Tunny/UI/OptimizeWindowTab/OutputTab.cs index cae36f03..64b51c57 100644 --- a/Tunny/UI/OptimizeWindowTab/OutputTab.cs +++ b/Tunny/UI/OptimizeWindowTab/OutputTab.cs @@ -37,6 +37,12 @@ private void RunOutputLoop(OutputMode mode) OutputLoop.StudyName = studyNameTextBox.Text; OutputLoop.NickNames = _component.GhInOut.Variables.Select(x => x.NickName).ToArray(); int[] indices = outputModelNumTextBox.Text.Split(',').Select(int.Parse).ToArray(); + SetOutputIndices(mode, indices); + outputResultBackgroundWorker.RunWorkerAsync(_component); + } + + private static void SetOutputIndices(OutputMode mode, int[] indices) + { switch (mode) { case OutputMode.ParatoSolutions: @@ -55,7 +61,6 @@ private void RunOutputLoop(OutputMode mode) default: throw new ArgumentException("Unsupported output mode."); } - outputResultBackgroundWorker.RunWorkerAsync(_component); } private static void CheckIndicesLength(int[] indices) From 9d44f51d9a2f57e8a7cd4f0ecd4c521051dd1809 Mon Sep 17 00:00:00 2001 From: hrntsm Date: Thu, 7 Jul 2022 20:46:51 +0900 Subject: [PATCH 08/11] Clean Optuna.cs --- Tunny/Solver/Optuna/{optuna.cs => Optuna.cs} | 44 ++++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) rename Tunny/Solver/Optuna/{optuna.cs => Optuna.cs} (89%) diff --git a/Tunny/Solver/Optuna/optuna.cs b/Tunny/Solver/Optuna/Optuna.cs similarity index 89% rename from Tunny/Solver/Optuna/optuna.cs rename to Tunny/Solver/Optuna/Optuna.cs index 7704c2b7..c3d44cf6 100644 --- a/Tunny/Solver/Optuna/optuna.cs +++ b/Tunny/Solver/Optuna/Optuna.cs @@ -162,29 +162,29 @@ public ModelResult[] GetModelResult(int[] resultNum, string studyName) private void SetTrialsToModelResult(int[] resultNum, List modelResult, dynamic study) { - switch (resultNum[0]) + if (resultNum[0] == -1) { - case -1: - var bestTrials = (dynamic[])study.best_trials; - foreach (dynamic trial in bestTrials) - { - ParseTrial(modelResult, trial); - } - break; - case -10: - var trials = (dynamic[])study.trials; - foreach (dynamic trial in trials) - { - ParseTrial(modelResult, trial); - } - break; - default: - foreach (int res in resultNum) - { - dynamic trial = study.trials[res]; - ParseTrial(modelResult, trial); - } - break; + var bestTrials = (dynamic[])study.best_trials; + foreach (dynamic trial in bestTrials) + { + ParseTrial(modelResult, trial); + } + } + else if (resultNum[0] == -10) + { + var trials = (dynamic[])study.trials; + foreach (dynamic trial in trials) + { + ParseTrial(modelResult, trial); + } + } + else + { + foreach (int res in resultNum) + { + dynamic trial = study.trials[res]; + ParseTrial(modelResult, trial); + } } } From 7a4943585684c59cb8bb31a9aa8d00314273c581 Mon Sep 17 00:00:00 2001 From: hrntsm Date: Thu, 7 Jul 2022 20:52:47 +0900 Subject: [PATCH 09/11] Fix Codacy issue --- Tunny/Optimization/OptimizeLoop.cs | 2 +- Tunny/Solver/Optuna/Sampler.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tunny/Optimization/OptimizeLoop.cs b/Tunny/Optimization/OptimizeLoop.cs index 79815d69..32969b97 100644 --- a/Tunny/Optimization/OptimizeLoop.cs +++ b/Tunny/Optimization/OptimizeLoop.cs @@ -17,7 +17,7 @@ internal static class OptimizeLoop { private static BackgroundWorker s_worker; private static TunnyComponent s_component; - public static TunnySettings Settings; + public static readonly TunnySettings Settings; internal static void RunMultiple(object sender, DoWorkEventArgs e) { diff --git a/Tunny/Solver/Optuna/Sampler.cs b/Tunny/Solver/Optuna/Sampler.cs index fb981f98..45b63e0e 100644 --- a/Tunny/Solver/Optuna/Sampler.cs +++ b/Tunny/Solver/Optuna/Sampler.cs @@ -6,7 +6,7 @@ namespace Tunny.Solver.Optuna { - public class Sampler + public static class Sampler { internal static dynamic Random(dynamic optuna, TunnySettings settings) { From eea4a20cfdfa8026788c020bbfa41103adffd8f3 Mon Sep 17 00:00:00 2001 From: hrntsm Date: Thu, 7 Jul 2022 20:57:10 +0900 Subject: [PATCH 10/11] Fix build error --- Tunny/Optimization/OptimizeLoop.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tunny/Optimization/OptimizeLoop.cs b/Tunny/Optimization/OptimizeLoop.cs index 32969b97..d09e352a 100644 --- a/Tunny/Optimization/OptimizeLoop.cs +++ b/Tunny/Optimization/OptimizeLoop.cs @@ -17,13 +17,12 @@ internal static class OptimizeLoop { private static BackgroundWorker s_worker; private static TunnyComponent s_component; - public static readonly TunnySettings Settings; + public static TunnySettings Settings; internal static void RunMultiple(object sender, DoWorkEventArgs e) { s_worker = sender as BackgroundWorker; s_component = e.Argument as TunnyComponent; - s_component.GhInOutInstantiate(); double[] result = RunOptimizationLoop(s_worker); From 4462f2def69479cb9e960c1f21add8b71fb3536e Mon Sep 17 00:00:00 2001 From: hrntsm Date: Thu, 7 Jul 2022 21:30:47 +0900 Subject: [PATCH 11/11] Add Optimze end state --- Tunny/Solver/Optuna/Algorithm.cs | 17 ++++++++++++++++- Tunny/Solver/Optuna/Optuna.cs | 24 ++++++++++++++++++++---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/Tunny/Solver/Optuna/Algorithm.cs b/Tunny/Solver/Optuna/Algorithm.cs index 6077d5d8..c87365d3 100644 --- a/Tunny/Solver/Optuna/Algorithm.cs +++ b/Tunny/Solver/Optuna/Algorithm.cs @@ -19,6 +19,7 @@ public class Algorithm private Func EvalFunc { get; set; } private double[] XOpt { get; set; } private double[] FxOpt { get; set; } + public EndState EndState { get; set; } = EndState.Error; public Algorithm( List variables, string[] objNickName, @@ -108,8 +109,14 @@ private void RunOptimize(int nTrials, double timeout, dynamic study, out double[ DateTime startTime = DateTime.Now; while (true) { - if (trialNum == nTrials || (DateTime.Now - startTime).TotalSeconds >= timeout) + if (trialNum >= nTrials) { + EndState = EndState.AllTrialFinish; + break; + } + else if ((DateTime.Now - startTime).TotalSeconds >= timeout) + { + EndState = EndState.Timeout; break; } @@ -216,5 +223,13 @@ public double[] GetFxOptimum() { return FxOpt; } + + } + + public enum EndState + { + AllTrialFinish, + Timeout, + Error } } diff --git a/Tunny/Solver/Optuna/Optuna.cs b/Tunny/Solver/Optuna/Optuna.cs index c3d44cf6..dd849abd 100644 --- a/Tunny/Solver/Optuna/Optuna.cs +++ b/Tunny/Solver/Optuna/Optuna.cs @@ -43,11 +43,11 @@ EvaluatedGHResult Eval(double[] x, int progress) try { - var tpe = new Algorithm(variables, objNickName, _settings, Eval); - tpe.Solve(); - XOpt = tpe.GetXOptimum(); + var optimize = new Algorithm(variables, objNickName, _settings, Eval); + optimize.Solve(); + XOpt = optimize.GetXOptimum(); - TunnyMessageBox.Show("Solver completed successfully.", "Tunny"); + ShowEndMessages(optimize); return true; } catch (Exception e) @@ -57,6 +57,22 @@ EvaluatedGHResult Eval(double[] x, int progress) } } + private static void ShowEndMessages(Algorithm optimize) + { + switch (optimize.EndState) + { + case EndState.Timeout: + TunnyMessageBox.Show("Solver completed successfully.\nThe specified time has elapsed.", "Tunny"); + break; + case EndState.AllTrialFinish: + TunnyMessageBox.Show("Solver completed successfully.\nThe specified number of trials has been completed.", "Tunny"); + break; + default: + TunnyMessageBox.Show("Solver error.", "Tunny"); + break; + } + } + private static void ShowErrorMessages(Exception e) { TunnyMessageBox.Show(