Clean Human in the loop code

CHANGELOG.md

@@ -18,6 +18,8 @@ Please see [here](https://github.com/hrntsm/Tunny/releases) for the data release
 - Default log file is set to "Verbose" with a retention period of 1 day.
 - Stop log output to console.
 - Disable window minimize button.
+- No log output to the Rhino console.
+- Human in the loop code was moved to the Optuna project to improve maintainability.
 
 ### Deprecated
 
@@ -32,6 +34,8 @@ for now removed features.
 - Tunny UI wont wake up when rhino7(net4.8) version settings file deserialize.
 - FishPrintByPath locks the image.
 - Mesh could not be put directly into Artifact.
+- Human in the loop exception with _stop_flag.
+- Even the multi-purpose Human in the loop specifies GP and the optimization does not flow, so TPE is chosen.
 
 ### Security
 

Tunny/Solver/HumanInTheLoop/HumanInTheLoopBase.cs → Optuna/Dashboard/HumanInTheLoop/HumanInTheLoopBase.cs

@@ -1,14 +1,9 @@
 using System.IO;
 using System.Text;
-using System.Windows.Forms;
 
 using Python.Runtime;
 
-using Tunny.Core.Settings;
-using Tunny.Core.Util;
-using Tunny.UI;
-
-namespace Tunny.Solver.HumanInTheLoop
+namespace Optuna.Dashboard.HumanInTheLoop
 {
     public class HumanInTheLoopBase
     {
@@ -17,14 +12,12 @@ public class HumanInTheLoopBase
 
         public HumanInTheLoopBase(string tmpPath, string storagePath)
         {
-            TLog.MethodStart();
             _tmpPath = tmpPath;
             _artifactPath = Path.Combine(Path.GetDirectoryName(storagePath), "artifacts");
         }
 
         public PyModule ImportBaseLibrary()
         {
-            TLog.MethodStart();
             PyModule library = Py.CreateScope();
             SetStdOutErrDirection(library);
             library.Import("os");
@@ -38,44 +31,29 @@ public PyModule ImportBaseLibrary()
 
         public void SetStdOutErrDirection(PyModule importedLibrary)
         {
-            TLog.MethodStart();
             string ioPath = Path.Combine(_tmpPath, "hitl.tmp");
             importedLibrary.Import("sys");
             importedLibrary.Exec("path = r'" + ioPath + "'");
             importedLibrary.Exec("sys.stdout = open(path, 'w', encoding='utf-8')");
             importedLibrary.Exec("sys.stderr = open(path, 'w', encoding='utf-8')");
         }
 
-        public static void WakeOptunaDashboard(Storage storage)
+        public static void WakeOptunaDashboard(string storagePath, string pythonPath)
         {
-            TLog.MethodStart();
-            if (File.Exists(storage.Path) == false)
+            if (File.Exists(storagePath) == false)
             {
-                ResultFileNotExistErrorMessage();
-                return;
+                throw new FileNotFoundException("Storage file not found.", storagePath);
             }
 
-            var dashboard = new Optuna.Dashboard.Handler(
-                Path.Combine(TEnvVariables.PythonPath, "Scripts", "optuna-dashboard.exe"),
-                storage.Path
+            var dashboard = new Handler(
+                Path.Combine(pythonPath, "Scripts", "optuna-dashboard.exe"),
+                storagePath
             );
             dashboard.Run(true);
         }
 
-        public static void ResultFileNotExistErrorMessage()
-        {
-            TLog.MethodStart();
-            TunnyMessageBox.Show(
-                "Please set exist result file path.",
-                "Error",
-                MessageBoxButtons.OK,
-                MessageBoxIcon.Error
-            );
-        }
-
         public static int GetRunningTrialNumber(dynamic study)
         {
-            TLog.MethodStart();
             PyModule lenModule = Py.CreateScope();
             var pyCode = new StringBuilder();
             pyCode.AppendLine("import optuna");
@@ -93,7 +71,6 @@ public static int GetRunningTrialNumber(dynamic study)
 
         public void CheckDirectoryIsExist()
         {
-            TLog.MethodStart();
             if (!Directory.Exists(_artifactPath))
             {
                 Directory.CreateDirectory(_artifactPath);

Tunny/Solver/HumanInTheLoop/HumanSliderInput.cs → Optuna/Dashboard/HumanInTheLoop/HumanSliderInput.cs

@@ -1,12 +1,11 @@
+using System.Collections.Generic;
 using System.Drawing;
 using System.Linq;
 using System.Text;
 
 using Python.Runtime;
 
-using Tunny.Core.Util;
-
-namespace Tunny.Solver.HumanInTheLoop
+namespace Optuna.Dashboard.HumanInTheLoop
 {
     public class HumanSliderInput : HumanInTheLoopBase
     {
@@ -15,7 +14,6 @@ public class HumanSliderInput : HumanInTheLoopBase
 
         public HumanSliderInput(string tmpPath, string storagePath) : base(tmpPath, storagePath)
         {
-            TLog.MethodStart();
             PyModule importedLibrary = ImportBaseLibrary();
             importedLibrary.Exec("from optuna_dashboard import save_note");
             importedLibrary.Exec("from optuna_dashboard import SliderWidget");
@@ -32,15 +30,13 @@ public HumanSliderInput(string tmpPath, string storagePath) : base(tmpPath, stor
 
         public void SetObjective(dynamic study, string[] objectiveNames)
         {
-            TLog.MethodStart();
             dynamic setObjectiveNames = _importedLibrary.Get("set_objective_names");
-            PyList pyNameList = PyConverter.EnumeratorToPyList(objectiveNames.Select(s => s.Replace("Human-in-the-Loop", "HITL")));
+            PyList pyNameList = EnumeratorToPyList(objectiveNames.Select(s => s.Replace("Human-in-the-Loop", "HITL")));
             setObjectiveNames(study, pyNameList);
         }
 
         public void SetWidgets(dynamic study, string[] objectiveNames)
         {
-            TLog.MethodStart();
             dynamic registerObjectiveFromWidgets = _importedLibrary.Get("register_objective_form_widgets");
             dynamic sliderWidget = _importedLibrary.Get("SliderWidget");
             dynamic objectiveUserAttrRef = _importedLibrary.Get("ObjectiveUserAttrRef");
@@ -64,7 +60,6 @@ public void SetWidgets(dynamic study, string[] objectiveNames)
 
         public void SaveNote(dynamic study, dynamic trial, Bitmap[] bitmaps)
         {
-            TLog.MethodStart();
             dynamic uploadArtifact = _importedLibrary.Get("upload_artifact");
             var noteText = new StringBuilder();
             noteText.AppendLine("# Image");
@@ -85,5 +80,15 @@ public void SaveNote(dynamic study, dynamic trial, Bitmap[] bitmaps)
             dynamic saveNote = _importedLibrary.Get("save_note");
             saveNote(trial, note);
         }
+
+        public static PyList EnumeratorToPyList(IEnumerable<string> enumerator)
+        {
+            var pyList = new PyList();
+            foreach (string item in enumerator)
+            {
+                pyList.Append(new PyString(item));
+            }
+            return pyList;
+        }
     }
 }

Tunny/Solver/HumanInTheLoop/Preferential.cs → Optuna/Dashboard/HumanInTheLoop/Preferential.cs

@@ -3,9 +3,7 @@
 
 using Python.Runtime;
 
-using Tunny.Core.Util;
-
-namespace Tunny.Solver.HumanInTheLoop
+namespace Optuna.Dashboard.HumanInTheLoop
 {
     public class Preferential : HumanInTheLoopBase
     {
@@ -15,7 +13,6 @@ public class Preferential : HumanInTheLoopBase
 
         public Preferential(string tmpPath, string storagePath) : base(tmpPath, storagePath)
         {
-            TLog.MethodStart();
             PyModule importedLibrary = ImportBaseLibrary();
             importedLibrary.Exec("from optuna_dashboard.artifact.file_system import FileSystemBackend");
             importedLibrary.Exec("from optuna_dashboard.artifact import upload_artifact");
@@ -30,7 +27,6 @@ public Preferential(string tmpPath, string storagePath) : base(tmpPath, storageP
 
         public dynamic CreateStudy(int nGenerate, string studyName, dynamic storage, string objectiveName)
         {
-            TLog.MethodStart();
             dynamic createStudy = _importedLibrary.Get("create_study");
             dynamic preferentialGPSampler = _importedLibrary.Get("PreferentialGPSampler");
             string name = studyName == null || studyName == "" ? "no-name-" + Guid.NewGuid().ToString("D") : studyName;
@@ -49,7 +45,6 @@ public dynamic CreateStudy(int nGenerate, string studyName, dynamic storage, str
 
         public void UploadArtifact(dynamic trial, Bitmap image)
         {
-            TLog.MethodStart();
             dynamic uploadArtifact = _importedLibrary.Get("upload_artifact");
             CheckDirectoryIsExist();
             string path = $"{_tmpPath}/image_{trial.number}.png";

Tunny/Solver/Algorithm.cs

@@ -2,11 +2,14 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
+using System.Reflection;
 using System.Text;
 using System.Threading;
 using System.Windows.Forms;
 
+using Optuna.Dashboard.HumanInTheLoop;
 using Optuna.Study;
+using Optuna.Util;
 
 using Python.Runtime;
 
@@ -103,7 +106,7 @@
         private void PreferentialOptimization(int nBatch, dynamic storage, dynamic artifactBackend, out Parameter[] parameter, out TrialGrasshopperItems result, out dynamic study)
         {
             TLog.MethodStart();
-            var preferentialOpt = new HumanInTheLoop.Preferential(TEnvVariables.TmpDirPath, Settings.Storage.Path);
+            var preferentialOpt = new Preferential(TEnvVariables.TmpDirPath, Settings.Storage.Path);
             if (Objective.Length > 1)
             {
                 TunnyMessageBox.Show("Human-in-the-Loop(Preferential GP optimization) only supports single objective optimization. Optimization is run without considering constraints.", "Tunny");
@@ -112,7 +115,7 @@
             study = preferentialOpt.CreateStudy(nBatch, Settings.StudyName, storage, objNickName[0]);
             var optInfo = new OptimizationHandlingInfo(int.MaxValue, 0, study, storage, artifactBackend, FishEgg, objNickName);
             SetStudyUserAttr(study, PyConverter.EnumeratorToPyList(Variables.Select(v => v.NickName)), false);
-            HumanInTheLoop.HumanInTheLoopBase.WakeOptunaDashboard(Settings.Storage);
+            HumanInTheLoopBase.WakeOptunaDashboard(Settings.Storage.Path, TEnvVariables.PythonPath);
             optInfo.Preferential = preferentialOpt;
             RunPreferentialOptimize(optInfo, nBatch, out parameter, out result);
         }
@@ -136,8 +139,8 @@
             string[] objNickName = Objective.GetNickNames();
             var optInfo = new OptimizationHandlingInfo(int.MaxValue, timeout, study, storage, artifactBackend, FishEgg, objNickName);
             SetStudyUserAttr(study, PyConverter.EnumeratorToPyList(Variables.Select(v => v.NickName)));
-            var humanSliderInput = new HumanInTheLoop.HumanSliderInput(TEnvVariables.TmpDirPath, Settings.Storage.Path);
-            HumanInTheLoop.HumanInTheLoopBase.WakeOptunaDashboard(Settings.Storage);
+            var humanSliderInput = new HumanSliderInput(TEnvVariables.TmpDirPath, Settings.Storage.Path);
+            HumanInTheLoopBase.WakeOptunaDashboard(Settings.Storage.Path, TEnvVariables.PythonPath);
             humanSliderInput.SetObjective(study, objNickName);
             humanSliderInput.SetWidgets(study, objNickName);
             optInfo.HumanSliderInput = humanSliderInput;
@@ -179,7 +182,7 @@
             if (nObjective == 1 && Objective.HumanInTheLoopType != HumanInTheLoopType.Preferential)
             {
                 PyObject[] pyBestParams = study.best_params.values();
                 string[] values = pyBestParams.Select(x => x.ToString()).ToArray();
                 string[] keys = (string[])study.best_params.keys();
                 var opt = new Parameter[Variables.Count];
 
@@ -253,7 +256,7 @@
                 {
                     break;
                 }
-                if (HumanInTheLoop.Preferential.GetRunningTrialNumber(optInfo.Study) >= nBatch)
+                if (Preferential.GetRunningTrialNumber(optInfo.Study) >= nBatch)
                 {
                     continue;
                 }
@@ -279,7 +282,7 @@
                 {
                     break;
                 }
-                if (HumanInTheLoop.HumanSliderInput.GetRunningTrialNumber(optInfo.Study) >= nBatch)
+                if (HumanSliderInput.GetRunningTrialNumber(optInfo.Study) >= nBatch)
                 {
                     continue;
                 }
@@ -290,112 +293,100 @@
             SaveInMemoryStudy(optInfo.Storage);
         }
 
         private TrialGrasshopperItems RunSingleOptimizeStep(OptimizationHandlingInfo optInfo, Parameter[] parameter, int trialNum, DateTime startTime)
         {
             TLog.MethodStart();
             dynamic trial;
             int progress = trialNum * 100 / optInfo.NTrials;
             var result = new TrialGrasshopperItems();
 
             int nullCount = 0;
 
             while (true)
             {
                 if (optInfo.Preferential != null && !optInfo.Study.should_generate() && !OptimizeLoop.IsForcedStopOptimize)
                 {
                     Thread.Sleep(100);
                     continue;
                 }
 
                 trial = optInfo.Study.ask();
                 SetOptimizationParameter(parameter, trial);
                 ProgressState pState = SetProgressState(optInfo, parameter, trialNum, startTime);
                 if (Settings.Optimize.IgnoreDuplicateSampling && IsSampleDuplicate(trial, out result))
                 {
                     TLog.Info($"Trial {trialNum} is duplicate sample.");
                     pState.IsReportOnly = true;
                     EvalFunc(pState, progress);
                     break;
                 }
                 else
                 {
                     result = EvalFunc(pState, progress);
                 }
 
                 if (optInfo.HumanSliderInput != null)
                 {
                     optInfo.HumanSliderInput.SaveNote(optInfo.Study, trial, result.Objectives.Images);
                 }
                 else if (optInfo.Preferential != null && result.Objectives.Images.Length == 1)
                 {
                     optInfo.Preferential.UploadArtifact(trial, result.Objectives.Images[0]);
                 }
 
                 if (nullCount >= 10)
                 {
                     return TenTimesNullResultErrorMessage();
                 }
                 else if (result.Objectives.Numbers.Contains(double.NaN))
                 {
                     nullCount++;
                 }
                 else
                 {
                     break;
                 }
             }
 
             SetTrialUserAttr(result, trial, optInfo);
             try
             {
                 if (result.Artifacts.Count() > 0)
                 {
                     result.Artifacts.UploadArtifacts(optInfo.ArtifactBackend, trial);
                 }
 
                 if (result.Attribute.TryGetValue("IsFAIL", out List<string> isFail) && isFail.Contains("True"))
                 {
                     dynamic optuna = Py.Import("optuna");
                     optInfo.Study.tell(trial, state: optuna.trial.TrialState.FAIL);
                     TLog.Warning($"Trial {trialNum} failed.");
                 }
                 else if (optInfo.HumanSliderInput == null && optInfo.Preferential == null)
                 {
                     optInfo.Study.tell(trial, result.Objectives.Numbers);
                     SetTrialResultLog(trialNum, result, optInfo, parameter);
                 }
             }
             catch (Exception e)
             {
                 TLog.Error(e.Message);
                 throw new ArgumentException(e.Message);
             }
             finally
             {
                 RunGC(result);
             }
 
             return result;
         }
 
         private static bool IsSampleDuplicate(dynamic trial, out TrialGrasshopperItems result)
         {
             double[] values;
             PyModule ps = Py.CreateScope();
-            ps.Exec(
-                "def check_duplicate(trial):\n" +
-                "    import optuna\n" +
-                "    from optuna.trial import TrialState\n" +
-                "    states_to_consider = (TrialState.COMPLETE,)\n" +
-                "    trials_to_consider = trial.study.get_trials(deepcopy=False, states=states_to_consider)\n" +
-                "    for t in reversed(trials_to_consider):\n" +
-                "        if trial.params == t.params:\n" +
-                "            trial.set_user_attr('NOTE', f'trial {t.number} and trial {trial.number} were duplicate parameters.')\n" +
-                "            if 'Constraint' in t.user_attrs:\n" +
-                "               trial.set_user_attr('Constraint', t.user_attrs['Constraint'])\n" +
-                "            return t.values\n" +
-                "    return None\n"
-            );
+            var assembly = Assembly.GetExecutingAssembly();
+            ps.Exec(ReadFileFromResource.Text(assembly, "Tunny.Solver.Python.check_duplication.py"));
             dynamic checkDuplicate = ps.Get("check_duplicate");
             values = checkDuplicate(trial);
             result = new TrialGrasshopperItems(values);

Tunny/Solver/OptimizationHandlingInfo.cs

@@ -1,5 +1,7 @@
 using System.Collections.Generic;
 
+using Optuna.Dashboard.HumanInTheLoop;
+
 using Tunny.Core.Util;
 using Tunny.Type;
 
@@ -14,8 +16,8 @@ sealed internal class OptimizationHandlingInfo
         public dynamic ArtifactBackend { get; set; }
         public Dictionary<string, FishEgg> EnqueueItems { get; set; }
         public string[] ObjectiveNames { get; set; }
-        public HumanInTheLoop.HumanSliderInput HumanSliderInput { get; set; }
-        public HumanInTheLoop.Preferential Preferential { get; set; }
+        public HumanSliderInput HumanSliderInput { get; set; }
+        public Preferential Preferential { get; set; }
 
         public OptimizationHandlingInfo(int nTrials, double timeout, dynamic study, dynamic storage,
                                         dynamic artifactBackend, Dictionary<string, FishEgg> enqueueItems,

Tunny/Solver/Python/check_duplication.py

@@ -0,0 +1,23 @@
+from optuna.trial import TrialState
+
+
+def check_duplicate(trial):
+
+    states_to_consider = (TrialState.COMPLETE,)
+    trials_to_consider = trial.study.get_trials(
+        deepcopy=False, states=states_to_consider
+    )
+    for t in reversed(trials_to_consider):
+        if trial.params == t.params:
+            set_attr_to_duplicate_trial(trial, t)
+            return t.values
+    return None
+
+
+def set_attr_to_duplicate_trial(base_trial, compared_trial) -> None:
+    base_trial.set_user_attr(
+        "NOTE",
+        f"trial {compared_trial.number} and trial {base_trial.number} were duplicate parameters.",
+    )
+    if "Constraint" in compared_trial.user_attrs:
+        base_trial.set_user_attr("Constraint", compared_trial.user_attrs["Constraint"])

Tunny/Tunny.csproj

@@ -29,6 +29,14 @@
     </None>
   </ItemGroup>
 
+  <ItemGroup>
+    <None Remove="Solver\Python\check_duplication.py" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <EmbeddedResource Include="Solver\Python\check_duplication.py" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Grasshopper" Version="7.13.21348.13001" IncludeAssets="compile;build" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />

Tunny/UI/OptimizationWindow.cs

@@ -20,7 +20,7 @@
     {
         private readonly OptimizeComponentBase _component;
         private readonly TSettings _settings;
         internal GrasshopperStates GrasshopperStatus;
 
         public OptimizationWindow(OptimizeComponentBase component)
         {
@@ -97,17 +97,6 @@
             outputResultBackgroundWorker.Dispose();
         }
 
-        private void UpdateGrasshopper(IList<Parameter> parameters)
-        {
-            TLog.MethodStart();
-            GrasshopperStatus = GrasshopperStates.RequestProcessing;
-
-            //Calculate Grasshopper
-            _component.GhInOut.NewSolution(parameters);
-
-            GrasshopperStatus = GrasshopperStates.RequestProcessed;
-        }
-
         private void OptimizationWindow_Load(object sender, EventArgs e)
         {
             TLog.MethodStart();
@@ -159,7 +148,10 @@
             {
                 TLog.Info("Set Tunny Human-in-the-loop mode");
                 Text = "Tunny (Human in the Loop mode)";
-                samplerComboBox.SelectedIndex = (int)SamplerType.GP; // GP
+                samplerComboBox.SelectedIndex = _component.GhInOut.Objectives.Length == 1
+                    ? (int)SamplerType.GP
+                    : (int)SamplerType.TPE;
+                tpeConstantLiarCheckBox.Checked = _component.GhInOut.Objectives.Length > 1;
                 samplerComboBox.Enabled = false;
                 inMemoryCheckBox.Checked = false;
                 inMemoryCheckBox.Enabled = false;