Feature/time out

CHANGELOG.md

@@ -21,6 +21,7 @@ Please see [here](https://github.com/hrntsm/Tunny/releases) for the data release
   - Components that output each attribute
 - Python package licenses to clearly state the license of each package.
 - requirements.txt file to avoid conflict python packages versions.
+- Implemented Timeout to stop optimization over time.
 
 ### Changed
 
@@ -36,7 +37,7 @@ Please see [here](https://github.com/hrntsm/Tunny/releases) for the data release
 - Stopped sampling when there was no geometry input
 - Once optimize output error, the component won't run again
   - I've tried to do a proper Dispose to fix this problem, but it still doesn't work sometimes.
-- Optuna-DashBoard and Delete Result Files functions do not work properly when a different Storage path is specified in Settings than the default.
+- Optuna-DashBoard and storage relate functions do not work properly when a different Storage path is specified in Settings than the default.
 
 ## [0.3.0] -2022-05-03
 

Tunny/Optimization/OptimizeLoop.cs

@@ -7,7 +7,7 @@
 
 using Tunny.Component;
 using Tunny.Settings;
-using Tunny.Solver;
+using Tunny.Solver.Optuna;
 using Tunny.UI;
 using Tunny.Util;
 
@@ -23,7 +23,6 @@ internal static void RunMultiple(object sender, DoWorkEventArgs e)
         {
             s_worker = sender as BackgroundWorker;
             s_component = e.Argument as TunnyComponent;
-
             s_component.GhInOutInstantiate();
 
             double[] result = RunOptimizationLoop(s_worker);
@@ -54,10 +53,9 @@ private static double[] RunOptimizationLoop(BackgroundWorker worker)
                 return new[] { double.NaN };
             }
 
-            var optunaSolver = new Optuna(s_component.GhInOut.ComponentFolder);
+            var optunaSolver = new Optuna(s_component.GhInOut.ComponentFolder, Settings);
 
-            bool solverStarted = optunaSolver.RunSolver(
-                variables, objectives, EvaluateFunction, Settings);
+            bool solverStarted = optunaSolver.RunSolver(variables, objectives, EvaluateFunction);
 
             return solverStarted ? optunaSolver.XOpt : new[] { double.NaN };
         }

Tunny/Optimization/OutputLoop.cs

@@ -6,7 +6,8 @@
 using Rhino.Geometry;
 
 using Tunny.Component;
-using Tunny.Solver;
+using Tunny.Settings;
+using Tunny.Solver.Optuna;
 using Tunny.Type;
 using Tunny.UI;
 using Tunny.Util;
@@ -17,6 +18,7 @@ internal static class OutputLoop
     {
         private static BackgroundWorker s_worker;
         private static TunnyComponent s_component;
+        public static TunnySettings Settings;
         public static string StudyName;
         public static string[] NickNames;
         public static int[] Indices;
@@ -29,7 +31,7 @@ internal static void Run(object sender, DoWorkEventArgs e)
 
             var fishes = new List<Fish>();
 
-            var optunaSolver = new Optuna(s_component.GhInOut.ComponentFolder);
+            var optunaSolver = new Optuna(s_component.GhInOut.ComponentFolder, Settings);
             ModelResult[] modelResult = optunaSolver.GetModelResult(Indices, StudyName);
             if (modelResult.Length == 0)
             {

Tunny/Settings/Optimize.cs

@@ -6,5 +6,6 @@ public class Optimize
         public int NumberOfTrials { get; set; } = 100;
         public bool LoadExistStudy { get; set; } = true;
         public int SelectSampler { get; set; }
+        public double Timeout { get; set; }
     }
 }

Tunny/Solver/Optuna/Algorithm.cs

@@ -0,0 +1,235 @@
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Text;
+
+using Python.Runtime;
+
+using Tunny.Optimization;
+using Tunny.Settings;
+using Tunny.Util;
+
+namespace Tunny.Solver.Optuna
+{
+    public class Algorithm
+    {
+        private List<Variable> Variables { get; set; }
+        private string[] ObjNickName { get; set; }
+        private TunnySettings Settings { get; set; }
+        private Func<double[], int, EvaluatedGHResult> EvalFunc { get; set; }
+        private double[] XOpt { get; set; }
+        private double[] FxOpt { get; set; }
+        public EndState EndState { get; set; } = EndState.Error;
+
+        public Algorithm(
+            List<Variable> variables, string[] objNickName,
+            TunnySettings settings,
+            Func<double[], int, EvaluatedGHResult> evalFunc)
+        {
+            Variables = variables;
+            ObjNickName = objNickName;
+            Settings = settings;
+            EvalFunc = evalFunc;
+
+        }
+
+        public void Solve()
+        {
+            int samplerType = Settings.Optimize.SelectSampler;
+            int nTrials = Settings.Optimize.NumberOfTrials;
+            double timeout = Settings.Optimize.Timeout <= 0 ? double.MaxValue : Settings.Optimize.Timeout;
+            int nObjective = ObjNickName.Length;
+            string[] directions = new string[nObjective];
+            for (int i = 0; i < nObjective; i++)
+            {
+                directions[i] = "minimize";
+            }
+
+            PythonEngine.Initialize();
+            using (Py.GIL())
+            {
+                dynamic optuna = Py.Import("optuna");
+                dynamic sampler = SetSamplerSettings(samplerType, ref nTrials, optuna);
+
+                dynamic study = optuna.create_study(
+                    sampler: sampler,
+                    directions: directions,
+                    storage: "sqlite:///" + Settings.Storage,
+                    study_name: Settings.StudyName,
+                    load_if_exists: Settings.Optimize.LoadExistStudy
+                );
+
+                var name = new StringBuilder();
+                foreach (string objName in ObjNickName)
+                {
+                    name.Append(objName + ",");
+                }
+                name.Remove(name.Length - 1, 1);
+                SetStudyUserAttr(study, name);
+
+                RunOptimize(nTrials, timeout, study, out double[] xTest, out EvaluatedGHResult result);
+                SetResultValues(nObjective, study, xTest, result);
+            }
+            PythonEngine.Shutdown();
+        }
+
+        private void SetResultValues(int nObjective, dynamic study, double[] xTest, EvaluatedGHResult result)
+        {
+            if (nObjective == 1)
+            {
+                double[] values = (double[])study.best_params.values();
+                string[] keys = (string[])study.best_params.keys();
+                double[] opt = new double[Variables.Count];
+
+                for (int i = 0; i < Variables.Count; i++)
+                {
+                    for (int j = 0; j < keys.Length; j++)
+                    {
+                        if (keys[j] == Variables[i].NickName)
+                        {
+                            opt[i] = values[j];
+                        }
+                    }
+                }
+                XOpt = opt;
+                FxOpt = new[] { (double)study.best_value };
+            }
+            else
+            {
+                XOpt = xTest;
+                FxOpt = result.ObjectiveValues.ToArray();
+            }
+        }
+
+        private void RunOptimize(int nTrials, double timeout, dynamic study, out double[] xTest, out EvaluatedGHResult result)
+        {
+            xTest = new double[Variables.Count];
+            result = new EvaluatedGHResult();
+            int trialNum = 0;
+            DateTime startTime = DateTime.Now;
+            while (true)
+            {
+                if (trialNum >= nTrials)
+                {
+                    EndState = EndState.AllTrialFinish;
+                    break;
+                }
+                else if ((DateTime.Now - startTime).TotalSeconds >= timeout)
+                {
+                    EndState = EndState.Timeout;
+                    break;
+                }
+
+                int progress = trialNum * 100 / nTrials;
+                dynamic trial = study.ask();
+
+                //TODO: Is this the correct way to handle the case of null?
+                int nullCount = 0;
+                while (nullCount < 10)
+                {
+                    for (int j = 0; j < Variables.Count; j++)
+                    {
+                        xTest[j] = trial.suggest_uniform(Variables[j].NickName, Variables[j].LowerBond, Variables[j].UpperBond);
+                    }
+                    result = EvalFunc(xTest, progress);
+
+                    if (result.ObjectiveValues.Contains(double.NaN))
+                    {
+                        trial = study.ask();
+                        nullCount++;
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+
+                SetTrialUserAttr(result, trial);
+                try
+                {
+                    study.tell(trial, result.ObjectiveValues.ToArray());
+                }
+                catch
+                {
+                }
+                trialNum++;
+            }
+        }
+
+        private static void SetStudyUserAttr(dynamic study, StringBuilder name)
+        {
+            study.set_user_attr("objective_names", name.ToString());
+            study.set_user_attr("tunny_version", Assembly.GetExecutingAssembly().GetName().Version.ToString(3));
+        }
+
+        private static void SetTrialUserAttr(EvaluatedGHResult result, dynamic trial)
+        {
+            if (result.GeometryJson.Count != 0)
+            {
+                var pyJson = new PyList();
+                foreach (string json in result.GeometryJson)
+                {
+                    pyJson.Append(new PyString(json));
+                }
+                trial.set_user_attr("Geometry", pyJson);
+            }
+
+            if (result.Attribute != null)
+            {
+                foreach (KeyValuePair<string, List<string>> pair in result.Attribute)
+                {
+                    var pyList = new PyList();
+                    foreach (string str in pair.Value)
+                    {
+                        pyList.Append(new PyString(str));
+                    }
+                    trial.set_user_attr(pair.Key, pyList);
+                }
+            }
+        }
+
+        private dynamic SetSamplerSettings(int samplerType, ref int nTrials, dynamic optuna)
+        {
+            dynamic sampler;
+            switch (samplerType)
+            {
+                case 0:
+                    sampler = Sampler.TPE(optuna, Settings);
+                    break;
+                case 1:
+                    sampler = Sampler.NSGAII(optuna, Settings);
+                    break;
+                case 2:
+                    sampler = Sampler.CmaEs(optuna, Settings);
+                    break;
+                case 3:
+                    sampler = Sampler.Random(optuna, Settings);
+                    break;
+                case 4:
+                    sampler = Sampler.Grid(optuna, Variables, ref nTrials);
+                    break;
+                default:
+                    throw new ArgumentException("Unknown sampler type");
+            }
+            return sampler;
+        }
+
+        public double[] GetXOptimum()
+        {
+            return XOpt;
+        }
+
+        public double[] GetFxOptimum()
+        {
+            return FxOpt;
+        }
+
+    }
+
+    public enum EndState
+    {
+        AllTrialFinish,
+        Timeout,
+        Error
+    }
+}