From a551947b65856d88f527ce695e49292ae7778565 Mon Sep 17 00:00:00 2001 From: NATSUME Hiroaki Date: Thu, 10 Oct 2024 17:42:16 +0900 Subject: [PATCH 1/6] Move Hitl dir to optuna.dashboard dir --- .../HumanInTheLoop/HumanInTheLoopBase.cs | 37 ++++--------------- .../HumanInTheLoop/HumanSliderInput.cs | 21 +++++++---- .../Dashboard}/HumanInTheLoop/Preferential.cs | 7 +--- Tunny/Solver/Algorithm.cs | 13 ++++--- Tunny/Solver/OptimizationHandlingInfo.cs | 6 ++- 5 files changed, 32 insertions(+), 52 deletions(-) rename {Tunny/Solver => Optuna/Dashboard}/HumanInTheLoop/HumanInTheLoopBase.cs (70%) rename {Tunny/Solver => Optuna/Dashboard}/HumanInTheLoop/HumanSliderInput.cs (88%) rename {Tunny/Solver => Optuna/Dashboard}/HumanInTheLoop/Preferential.cs (94%) diff --git a/Tunny/Solver/HumanInTheLoop/HumanInTheLoopBase.cs b/Optuna/Dashboard/HumanInTheLoop/HumanInTheLoopBase.cs similarity index 70% rename from Tunny/Solver/HumanInTheLoop/HumanInTheLoopBase.cs rename to Optuna/Dashboard/HumanInTheLoop/HumanInTheLoopBase.cs index db68a8c0..63320eae 100644 --- a/Tunny/Solver/HumanInTheLoop/HumanInTheLoopBase.cs +++ b/Optuna/Dashboard/HumanInTheLoop/HumanInTheLoopBase.cs @@ -1,14 +1,9 @@ using System.IO; using System.Text; -using System.Windows.Forms; using Python.Runtime; -using Tunny.Core.Settings; -using Tunny.Core.Util; -using Tunny.UI; - -namespace Tunny.Solver.HumanInTheLoop +namespace Optuna.Dashboard.HumanInTheLoop { public class HumanInTheLoopBase { @@ -17,14 +12,12 @@ public class HumanInTheLoopBase public HumanInTheLoopBase(string tmpPath, string storagePath) { - TLog.MethodStart(); _tmpPath = tmpPath; _artifactPath = Path.Combine(Path.GetDirectoryName(storagePath), "artifacts"); } public PyModule ImportBaseLibrary() { - TLog.MethodStart(); PyModule library = Py.CreateScope(); SetStdOutErrDirection(library); library.Import("os"); @@ -38,7 +31,6 @@ public PyModule ImportBaseLibrary() public void SetStdOutErrDirection(PyModule importedLibrary) { - TLog.MethodStart(); string ioPath = Path.Combine(_tmpPath, "hitl.tmp"); importedLibrary.Import("sys"); importedLibrary.Exec("path = r'" + ioPath + "'"); @@ -46,36 +38,22 @@ public void SetStdOutErrDirection(PyModule importedLibrary) importedLibrary.Exec("sys.stderr = open(path, 'w', encoding='utf-8')"); } - public static void WakeOptunaDashboard(Storage storage) + public static void WakeOptunaDashboard(string storagePath, string pythonPath) { - TLog.MethodStart(); - if (File.Exists(storage.Path) == false) + if (File.Exists(storagePath) == false) { - ResultFileNotExistErrorMessage(); - return; + throw new FileNotFoundException("Storage file not found.", storagePath); } - var dashboard = new Optuna.Dashboard.Handler( - Path.Combine(TEnvVariables.PythonPath, "Scripts", "optuna-dashboard.exe"), - storage.Path + var dashboard = new Handler( + Path.Combine(pythonPath, "Scripts", "optuna-dashboard.exe"), + storagePath ); dashboard.Run(true); } - public static void ResultFileNotExistErrorMessage() - { - TLog.MethodStart(); - TunnyMessageBox.Show( - "Please set exist result file path.", - "Error", - MessageBoxButtons.OK, - MessageBoxIcon.Error - ); - } - public static int GetRunningTrialNumber(dynamic study) { - TLog.MethodStart(); PyModule lenModule = Py.CreateScope(); var pyCode = new StringBuilder(); pyCode.AppendLine("import optuna"); @@ -93,7 +71,6 @@ public static int GetRunningTrialNumber(dynamic study) public void CheckDirectoryIsExist() { - TLog.MethodStart(); if (!Directory.Exists(_artifactPath)) { Directory.CreateDirectory(_artifactPath); diff --git a/Tunny/Solver/HumanInTheLoop/HumanSliderInput.cs b/Optuna/Dashboard/HumanInTheLoop/HumanSliderInput.cs similarity index 88% rename from Tunny/Solver/HumanInTheLoop/HumanSliderInput.cs rename to Optuna/Dashboard/HumanInTheLoop/HumanSliderInput.cs index 3cacb0e9..52dbe848 100644 --- a/Tunny/Solver/HumanInTheLoop/HumanSliderInput.cs +++ b/Optuna/Dashboard/HumanInTheLoop/HumanSliderInput.cs @@ -1,12 +1,11 @@ +using System.Collections.Generic; using System.Drawing; using System.Linq; using System.Text; using Python.Runtime; -using Tunny.Core.Util; - -namespace Tunny.Solver.HumanInTheLoop +namespace Optuna.Dashboard.HumanInTheLoop { public class HumanSliderInput : HumanInTheLoopBase { @@ -15,7 +14,6 @@ public class HumanSliderInput : HumanInTheLoopBase public HumanSliderInput(string tmpPath, string storagePath) : base(tmpPath, storagePath) { - TLog.MethodStart(); PyModule importedLibrary = ImportBaseLibrary(); importedLibrary.Exec("from optuna_dashboard import save_note"); importedLibrary.Exec("from optuna_dashboard import SliderWidget"); @@ -32,15 +30,13 @@ public HumanSliderInput(string tmpPath, string storagePath) : base(tmpPath, stor public void SetObjective(dynamic study, string[] objectiveNames) { - TLog.MethodStart(); dynamic setObjectiveNames = _importedLibrary.Get("set_objective_names"); - PyList pyNameList = PyConverter.EnumeratorToPyList(objectiveNames.Select(s => s.Replace("Human-in-the-Loop", "HITL"))); + PyList pyNameList = EnumeratorToPyList(objectiveNames.Select(s => s.Replace("Human-in-the-Loop", "HITL"))); setObjectiveNames(study, pyNameList); } public void SetWidgets(dynamic study, string[] objectiveNames) { - TLog.MethodStart(); dynamic registerObjectiveFromWidgets = _importedLibrary.Get("register_objective_form_widgets"); dynamic sliderWidget = _importedLibrary.Get("SliderWidget"); dynamic objectiveUserAttrRef = _importedLibrary.Get("ObjectiveUserAttrRef"); @@ -64,7 +60,6 @@ public void SetWidgets(dynamic study, string[] objectiveNames) public void SaveNote(dynamic study, dynamic trial, Bitmap[] bitmaps) { - TLog.MethodStart(); dynamic uploadArtifact = _importedLibrary.Get("upload_artifact"); var noteText = new StringBuilder(); noteText.AppendLine("# Image"); @@ -85,5 +80,15 @@ public void SaveNote(dynamic study, dynamic trial, Bitmap[] bitmaps) dynamic saveNote = _importedLibrary.Get("save_note"); saveNote(trial, note); } + + public static PyList EnumeratorToPyList(IEnumerable enumerator) + { + var pyList = new PyList(); + foreach (string item in enumerator) + { + pyList.Append(new PyString(item)); + } + return pyList; + } } } diff --git a/Tunny/Solver/HumanInTheLoop/Preferential.cs b/Optuna/Dashboard/HumanInTheLoop/Preferential.cs similarity index 94% rename from Tunny/Solver/HumanInTheLoop/Preferential.cs rename to Optuna/Dashboard/HumanInTheLoop/Preferential.cs index 612de6ab..9ff924c3 100644 --- a/Tunny/Solver/HumanInTheLoop/Preferential.cs +++ b/Optuna/Dashboard/HumanInTheLoop/Preferential.cs @@ -3,9 +3,7 @@ using Python.Runtime; -using Tunny.Core.Util; - -namespace Tunny.Solver.HumanInTheLoop +namespace Optuna.Dashboard.HumanInTheLoop { public class Preferential : HumanInTheLoopBase { @@ -15,7 +13,6 @@ public class Preferential : HumanInTheLoopBase public Preferential(string tmpPath, string storagePath) : base(tmpPath, storagePath) { - TLog.MethodStart(); PyModule importedLibrary = ImportBaseLibrary(); importedLibrary.Exec("from optuna_dashboard.artifact.file_system import FileSystemBackend"); importedLibrary.Exec("from optuna_dashboard.artifact import upload_artifact"); @@ -30,7 +27,6 @@ public Preferential(string tmpPath, string storagePath) : base(tmpPath, storageP public dynamic CreateStudy(int nGenerate, string studyName, dynamic storage, string objectiveName) { - TLog.MethodStart(); dynamic createStudy = _importedLibrary.Get("create_study"); dynamic preferentialGPSampler = _importedLibrary.Get("PreferentialGPSampler"); string name = studyName == null || studyName == "" ? "no-name-" + Guid.NewGuid().ToString("D") : studyName; @@ -49,7 +45,6 @@ public dynamic CreateStudy(int nGenerate, string studyName, dynamic storage, str public void UploadArtifact(dynamic trial, Bitmap image) { - TLog.MethodStart(); dynamic uploadArtifact = _importedLibrary.Get("upload_artifact"); CheckDirectoryIsExist(); string path = $"{_tmpPath}/image_{trial.number}.png"; diff --git a/Tunny/Solver/Algorithm.cs b/Tunny/Solver/Algorithm.cs index d4a3f741..2b9c03cd 100644 --- a/Tunny/Solver/Algorithm.cs +++ b/Tunny/Solver/Algorithm.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Windows.Forms; +using Optuna.Dashboard.HumanInTheLoop; using Optuna.Study; using Python.Runtime; @@ -103,7 +104,7 @@ public void Solve() private void PreferentialOptimization(int nBatch, dynamic storage, dynamic artifactBackend, out Parameter[] parameter, out TrialGrasshopperItems result, out dynamic study) { TLog.MethodStart(); - var preferentialOpt = new HumanInTheLoop.Preferential(TEnvVariables.TmpDirPath, Settings.Storage.Path); + var preferentialOpt = new Preferential(TEnvVariables.TmpDirPath, Settings.Storage.Path); if (Objective.Length > 1) { TunnyMessageBox.Show("Human-in-the-Loop(Preferential GP optimization) only supports single objective optimization. Optimization is run without considering constraints.", "Tunny"); @@ -112,7 +113,7 @@ private void PreferentialOptimization(int nBatch, dynamic storage, dynamic artif study = preferentialOpt.CreateStudy(nBatch, Settings.StudyName, storage, objNickName[0]); var optInfo = new OptimizationHandlingInfo(int.MaxValue, 0, study, storage, artifactBackend, FishEgg, objNickName); SetStudyUserAttr(study, PyConverter.EnumeratorToPyList(Variables.Select(v => v.NickName)), false); - HumanInTheLoop.HumanInTheLoopBase.WakeOptunaDashboard(Settings.Storage); + HumanInTheLoopBase.WakeOptunaDashboard(Settings.Storage.Path, TEnvVariables.PythonPath); optInfo.Preferential = preferentialOpt; RunPreferentialOptimize(optInfo, nBatch, out parameter, out result); } @@ -136,8 +137,8 @@ private void HumanSliderInputOptimization(int nBatch, double timeout, string[] d string[] objNickName = Objective.GetNickNames(); var optInfo = new OptimizationHandlingInfo(int.MaxValue, timeout, study, storage, artifactBackend, FishEgg, objNickName); SetStudyUserAttr(study, PyConverter.EnumeratorToPyList(Variables.Select(v => v.NickName))); - var humanSliderInput = new HumanInTheLoop.HumanSliderInput(TEnvVariables.TmpDirPath, Settings.Storage.Path); - HumanInTheLoop.HumanInTheLoopBase.WakeOptunaDashboard(Settings.Storage); + var humanSliderInput = new HumanSliderInput(TEnvVariables.TmpDirPath, Settings.Storage.Path); + HumanInTheLoopBase.WakeOptunaDashboard(Settings.Storage.Path, TEnvVariables.PythonPath); humanSliderInput.SetObjective(study, objNickName); humanSliderInput.SetWidgets(study, objNickName); optInfo.HumanSliderInput = humanSliderInput; @@ -253,7 +254,7 @@ private void RunPreferentialOptimize(OptimizationHandlingInfo optInfo, int nBatc { break; } - if (HumanInTheLoop.Preferential.GetRunningTrialNumber(optInfo.Study) >= nBatch) + if (Preferential.GetRunningTrialNumber(optInfo.Study) >= nBatch) { continue; } @@ -279,7 +280,7 @@ private void RunHumanSidlerInputOptimize(OptimizationHandlingInfo optInfo, int n { break; } - if (HumanInTheLoop.HumanSliderInput.GetRunningTrialNumber(optInfo.Study) >= nBatch) + if (HumanSliderInput.GetRunningTrialNumber(optInfo.Study) >= nBatch) { continue; } diff --git a/Tunny/Solver/OptimizationHandlingInfo.cs b/Tunny/Solver/OptimizationHandlingInfo.cs index 9ba7cb1a..6fd499d2 100644 --- a/Tunny/Solver/OptimizationHandlingInfo.cs +++ b/Tunny/Solver/OptimizationHandlingInfo.cs @@ -1,5 +1,7 @@ using System.Collections.Generic; +using Optuna.Dashboard.HumanInTheLoop; + using Tunny.Core.Util; using Tunny.Type; @@ -14,8 +16,8 @@ sealed internal class OptimizationHandlingInfo public dynamic ArtifactBackend { get; set; } public Dictionary EnqueueItems { get; set; } public string[] ObjectiveNames { get; set; } - public HumanInTheLoop.HumanSliderInput HumanSliderInput { get; set; } - public HumanInTheLoop.Preferential Preferential { get; set; } + public HumanSliderInput HumanSliderInput { get; set; } + public Preferential Preferential { get; set; } public OptimizationHandlingInfo(int nTrials, double timeout, dynamic study, dynamic storage, dynamic artifactBackend, Dictionary enqueueItems, From 5895e484ded7a35dcb851710d1e1d7c06e1d538e Mon Sep 17 00:00:00 2001 From: NATSUME Hiroaki Date: Thu, 10 Oct 2024 17:59:54 +0900 Subject: [PATCH 2/6] Refactor code to use separate Python file for duplicate parameter check --- Tunny/Solver/Algorithm.cs | 18 ++++-------------- Tunny/Solver/Python/check_duplication.py | 19 +++++++++++++++++++ Tunny/Tunny.csproj | 8 ++++++++ 3 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 Tunny/Solver/Python/check_duplication.py diff --git a/Tunny/Solver/Algorithm.cs b/Tunny/Solver/Algorithm.cs index 2b9c03cd..d7b02451 100644 --- a/Tunny/Solver/Algorithm.cs +++ b/Tunny/Solver/Algorithm.cs @@ -2,12 +2,14 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Reflection; using System.Text; using System.Threading; using System.Windows.Forms; using Optuna.Dashboard.HumanInTheLoop; using Optuna.Study; +using Optuna.Util; using Python.Runtime; @@ -383,20 +385,8 @@ private static bool IsSampleDuplicate(dynamic trial, out TrialGrasshopperItems r { double[] values; PyModule ps = Py.CreateScope(); - ps.Exec( - "def check_duplicate(trial):\n" + - " import optuna\n" + - " from optuna.trial import TrialState\n" + - " states_to_consider = (TrialState.COMPLETE,)\n" + - " trials_to_consider = trial.study.get_trials(deepcopy=False, states=states_to_consider)\n" + - " for t in reversed(trials_to_consider):\n" + - " if trial.params == t.params:\n" + - " trial.set_user_attr('NOTE', f'trial {t.number} and trial {trial.number} were duplicate parameters.')\n" + - " if 'Constraint' in t.user_attrs:\n" + - " trial.set_user_attr('Constraint', t.user_attrs['Constraint'])\n" + - " return t.values\n" + - " return None\n" - ); + var assembly = Assembly.GetExecutingAssembly(); + ps.Exec(ReadFileFromResource.Text(assembly, "Tunny.Solver.Python.check_duplication.py")); dynamic checkDuplicate = ps.Get("check_duplicate"); values = checkDuplicate(trial); result = new TrialGrasshopperItems(values); diff --git a/Tunny/Solver/Python/check_duplication.py b/Tunny/Solver/Python/check_duplication.py new file mode 100644 index 00000000..8f00b5a6 --- /dev/null +++ b/Tunny/Solver/Python/check_duplication.py @@ -0,0 +1,19 @@ +from optuna.trial import TrialState + + +def check_duplicate(trial): + + states_to_consider = (TrialState.COMPLETE,) + trials_to_consider = trial.study.get_trials( + deepcopy=False, states=states_to_consider + ) + for t in reversed(trials_to_consider): + if trial.params == t.params: + trial.set_user_attr( + "NOTE", + f"trial {t.number} and trial {trial.number} were duplicate parameters.", + ) + if "Constraint" in t.user_attrs: + trial.set_user_attr("Constraint", t.user_attrs["Constraint"]) + return t.values + return None diff --git a/Tunny/Tunny.csproj b/Tunny/Tunny.csproj index 9cf0dbbe..b9f467a8 100644 --- a/Tunny/Tunny.csproj +++ b/Tunny/Tunny.csproj @@ -29,6 +29,14 @@ + + + + + + + + From 1a6181e3cf0747ae70079bee6aef4738f7322151 Mon Sep 17 00:00:00 2001 From: NATSUME Hiroaki Date: Thu, 10 Oct 2024 18:21:19 +0900 Subject: [PATCH 3/6] Fix MO Human-in-the-loop mode sampler to TPE --- Tunny/UI/OptimizationWindow.cs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/Tunny/UI/OptimizationWindow.cs b/Tunny/UI/OptimizationWindow.cs index 8125517d..141b1c89 100644 --- a/Tunny/UI/OptimizationWindow.cs +++ b/Tunny/UI/OptimizationWindow.cs @@ -97,17 +97,6 @@ public void BGDispose() outputResultBackgroundWorker.Dispose(); } - private void UpdateGrasshopper(IList parameters) - { - TLog.MethodStart(); - GrasshopperStatus = GrasshopperStates.RequestProcessing; - - //Calculate Grasshopper - _component.GhInOut.NewSolution(parameters); - - GrasshopperStatus = GrasshopperStates.RequestProcessed; - } - private void OptimizationWindow_Load(object sender, EventArgs e) { TLog.MethodStart(); @@ -159,7 +148,10 @@ private void SetUIValues() { TLog.Info("Set Tunny Human-in-the-loop mode"); Text = "Tunny (Human in the Loop mode)"; - samplerComboBox.SelectedIndex = (int)SamplerType.GP; // GP + samplerComboBox.SelectedIndex = _component.GhInOut.Objectives.Length == 1 + ? (int)SamplerType.GP + : (int)SamplerType.TPE; + tpeConstantLiarCheckBox.Checked = _component.GhInOut.Objectives.Length > 1; samplerComboBox.Enabled = false; inMemoryCheckBox.Checked = false; inMemoryCheckBox.Enabled = false; From ac5831c607b98ddfc1876d30d834bdd20e67ae26 Mon Sep 17 00:00:00 2001 From: NATSUME Hiroaki Date: Thu, 10 Oct 2024 18:25:27 +0900 Subject: [PATCH 4/6] Update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2495503b..cd845c80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Please see [here](https://github.com/hrntsm/Tunny/releases) for the data release - Default log file is set to "Verbose" with a retention period of 1 day. - Stop log output to console. - Disable window minimize button. +- No log output to the Rhino console. ### Deprecated @@ -32,6 +33,8 @@ for now removed features. - Tunny UI wont wake up when rhino7(net4.8) version settings file deserialize. - FishPrintByPath locks the image. - Mesh could not be put directly into Artifact. +- Human in the loop exception with _stop_flag. +- Even the multi-purpose Human in the loop specifies GP and the optimization does not flow, so TPE is chosen. ### Security From 6484c77f8374096860808cbdd10f9e8f055fe5d8 Mon Sep 17 00:00:00 2001 From: NATSUME Hiroaki Date: Thu, 10 Oct 2024 18:26:38 +0900 Subject: [PATCH 5/6] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd845c80..0c03d862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Please see [here](https://github.com/hrntsm/Tunny/releases) for the data release - Stop log output to console. - Disable window minimize button. - No log output to the Rhino console. +- Human in the loop code was moved to the Optuna project to improve maintainability. ### Deprecated From 7748eadbe79bf0ba85b8b0d21856b23b098b99f0 Mon Sep 17 00:00:00 2001 From: NATSUME Hiroaki Date: Thu, 10 Oct 2024 18:32:54 +0900 Subject: [PATCH 6/6] Clean code --- Tunny/Solver/Python/check_duplication.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Tunny/Solver/Python/check_duplication.py b/Tunny/Solver/Python/check_duplication.py index 8f00b5a6..d714370d 100644 --- a/Tunny/Solver/Python/check_duplication.py +++ b/Tunny/Solver/Python/check_duplication.py @@ -9,11 +9,15 @@ def check_duplicate(trial): ) for t in reversed(trials_to_consider): if trial.params == t.params: - trial.set_user_attr( - "NOTE", - f"trial {t.number} and trial {trial.number} were duplicate parameters.", - ) - if "Constraint" in t.user_attrs: - trial.set_user_attr("Constraint", t.user_attrs["Constraint"]) + set_attr_to_duplicate_trial(trial, t) return t.values return None + + +def set_attr_to_duplicate_trial(base_trial, compared_trial) -> None: + base_trial.set_user_attr( + "NOTE", + f"trial {compared_trial.number} and trial {base_trial.number} were duplicate parameters.", + ) + if "Constraint" in compared_trial.user_attrs: + base_trial.set_user_attr("Constraint", compared_trial.user_attrs["Constraint"])