From 98e61ffd4019ac7c226ab12ea31796ce25c75f77 Mon Sep 17 00:00:00 2001 From: hrntsm Date: Sat, 24 Feb 2024 00:01:45 +0900 Subject: [PATCH 1/4] Fix clustering plot --- Optuna/Visualization/Visualization.cs | 115 +++++++++++++++++++++ Tunny/Solver/Visualize.cs | 12 ++- Tunny/UI/OptimizeWindowTab/VisualizeTab.cs | 20 +++- 3 files changed, 145 insertions(+), 2 deletions(-) diff --git a/Optuna/Visualization/Visualization.cs b/Optuna/Visualization/Visualization.cs index 0221738c..991777fc 100644 --- a/Optuna/Visualization/Visualization.cs +++ b/Optuna/Visualization/Visualization.cs @@ -171,6 +171,121 @@ public void Hypervolume() _fig = visualize(_study); } + public void Clustering(int nClusters, string targetType, int targetIndex) + { + PyModule ps = Py.CreateScope(); + ps.Exec( + "def visualize(study, n_clusters, target_type, target_index):\n" + + " import numpy as np\n" + + " import optuna\n" + + " from sklearn.cluster import KMeans\n" + + " import plotly.graph_objects as go\n" + + " from optuna.visualization._utils import _make_hovertext\n" + + + " trials = study.get_trials(deepcopy=False, states=[optuna.trial.TrialState.COMPLETE])\n" + + " feasible_trials = []\n" + + " infeasible_trials = []\n" + + " for trial in trials:\n" + + " constraints = trial.system_attrs.get('constraints')\n" + + " if constraints is None or all([x <= 0.0 for x in constraints]):\n" + + " feasible_trials.append(trial)\n" + + " else:\n" + + " infeasible_trials.append(trial)\n" + + + " target = []\n" + + " if target_type == 'objective':\n" + + " target = [trial.values[target_index] for trial in feasible_trials]\n" + + " else:\n" + + " target = [\n" + + " list(trial.params.values())[target_index] for trial in feasible_trials\n" + + " ]\n" + + " np_array = np.array(target).reshape(-1, 1)\n" + + " kmeans = KMeans(n_clusters=n_clusters).fit(np_array)\n" + + + " feasible_marker = dict(\n" + + " color=kmeans.labels_,\n" + + " showscale=True,\n" + + " colorscale='RdYlBu_r',\n" + + " colorbar=dict(title='Cluster'),\n" + + " size=12,\n" + + " )\n" + + " infeasible_marker = dict(\n" + + " color='#cccccc',\n" + + " showscale=False,\n" + + " size=12,\n" + + " )\n" + + " fig = go.Figure()\n" + + " if len(study.directions) == 2:\n" + + " fig.add_trace(\n" + + " go.Scatter(\n" + + " x=[trial.values[0] for trial in feasible_trials],\n" + + " y=[trial.values[1] for trial in feasible_trials],\n" + + " mode='markers',\n" + + " marker=feasible_marker,\n" + + " showlegend=False,\n" + + " text=[_make_hovertext(trial) for trial in feasible_trials],\n" + + " hovertemplate='%{text}Trial',\n" + + " )\n" + + " )\n" + + " fig.add_trace(\n" + + " go.Scatter(\n" + + " x=[trial.values[0] for trial in infeasible_trials],\n" + + " y=[trial.values[1] for trial in infeasible_trials],\n" + + " mode='markers',\n" + + " marker=infeasible_marker,\n" + + " showlegend=False,\n" + + " text=[_make_hovertext(trial) for trial in feasible_trials],\n" + + " hovertemplate='%{text}Infeasible Trial',\n" + + " )\n" + + " )\n" + + " else:\n" + + " fig.add_trace(\n" + + " go.Scatter3d(\n" + + " x=[trial.values[0] for trial in feasible_trials],\n" + + " y=[trial.values[1] for trial in feasible_trials],\n" + + " z=[trial.values[2] for trial in feasible_trials],\n" + + " mode='markers',\n" + + " marker=feasible_marker,\n" + + " showlegend=False,\n" + + " text=[_make_hovertext(trial) for trial in feasible_trials],\n" + + " hovertemplate='%{text}Trial',\n" + + " )\n" + + " )\n" + + " fig.add_trace(\n" + + " go.Scatter3d(\n" + + " x=[trial.values[0] for trial in infeasible_trials],\n" + + " y=[trial.values[1] for trial in infeasible_trials],\n" + + " z=[trial.values[2] for trial in infeasible_trials],\n" + + " mode='markers',\n" + + " marker=infeasible_marker,\n" + + " showlegend=False,\n" + + " text=[_make_hovertext(trial) for trial in feasible_trials],\n" + + " hovertemplate='%{text}Infeasible Trial',\n" + + " )\n" + + " )\n" + + " metric_names = study.metric_names\n" + + " if metric_names is not None:\n" + + " if len(metric_names) == 3:\n" + + " fig.update_layout(\n" + + " title=f'Clustering of Trials',\n" + + " scene=dict(\n" + + " xaxis_title=metric_names[0],\n" + + " yaxis_title=metric_names[1],\n" + + " zaxis_title=metric_names[2],\n" + + " ),\n" + + " )\n" + + " else:\n" + + " fig.update_layout(\n" + + " title=f'Clustering of Trials',\n" + + " xaxis=dict(title=metric_names[0]),\n" + + " yaxis=dict(title=metric_names[1]),\n" + + " )\n" + + " return go.Figure(fig)\n" + ); + dynamic visualize = ps.Get("visualize"); + _fig = visualize(_study, nClusters, targetType, targetIndex); + } + public void TruncateParetoFrontPlotHover() { CheckPlotCreated(); diff --git a/Tunny/Solver/Visualize.cs b/Tunny/Solver/Visualize.cs index f71bcada..2205b81b 100644 --- a/Tunny/Solver/Visualize.cs +++ b/Tunny/Solver/Visualize.cs @@ -85,7 +85,7 @@ private Visualization CreateFigure(dynamic study, Plot pSettings) visualize.ParallelCoordinate(pSettings.TargetObjectiveName[0], pSettings.TargetObjectiveIndex[0], pSettings.TargetVariableName); break; case "param importances": - visualize.ParamImportances(pSettings.TargetObjectiveName[0], pSettings.TargetObjectiveIndex[0]); + visualize.ParamImportances(pSettings.TargetObjectiveName[0], pSettings.TargetObjectiveIndex[0]); break; case "pareto front": visualize.ParetoFront(pSettings.TargetObjectiveName, pSettings.TargetObjectiveIndex, _hasConstraint, pSettings.IncludeDominatedTrials); @@ -97,6 +97,16 @@ private Visualization CreateFigure(dynamic study, Plot pSettings) case "hypervolume": visualize.Hypervolume(); break; + case "clustering": + if (pSettings.TargetObjectiveIndex.Length > 0) + { + visualize.Clustering(pSettings.ClusterCount, "objective", pSettings.TargetObjectiveIndex[0]); + } + else + { + visualize.Clustering(pSettings.ClusterCount, "variable", pSettings.TargetVariableIndex[0]); + } + break; default: TunnyMessageBox.Show("This visualization type is not supported in this study case.", "Tunny"); break; diff --git a/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs b/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs index b6d1b006..7e7d67b5 100644 --- a/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs +++ b/Tunny/UI/OptimizeWindowTab/VisualizeTab.cs @@ -119,13 +119,31 @@ private static bool CheckTargetValues(Plot pSettings) case "slice": return CheckOneObjSomeVarTargets(pSettings); case "pareto front": - case "clustering": return CheckParetoFrontTargets(pSettings); + case "clustering": + return CheckClusteringTargets(pSettings); default: return CheckOneObjectives(pSettings); } } + private static bool CheckClusteringTargets(Plot pSettings) + { + TLog.MethodStart(); + bool result = true; + if (pSettings.TargetObjectiveIndex.Length > 0 && pSettings.TargetVariableIndex.Length > 0) + { + TunnyMessageBox.Show("Both variables and objects are selected, but the objects are clustered in the target.\nIf you want to target variables, deselect the objectivesThis visualization type is not supported in this study case.", "Tunny"); + } + if (pSettings.TargetObjectiveName.Length == 0 && pSettings.TargetVariableName.Length == 0) + { + TunnyMessageBox.Show("Please select one or more.", "Tunny"); + result = false; + } + + return result; + } + private static bool CheckOneObjectives(Plot pSettings) { TLog.MethodStart(); From f659b53354465dcc204b00386bc14cbe0a5d717a Mon Sep 17 00:00:00 2001 From: hrntsm Date: Sat, 24 Feb 2024 12:50:27 +0900 Subject: [PATCH 2/4] Remove Hypervolume.cs --- CHANGELOG.md | 2 +- Tunny/Solver/Algorithm.cs | 10 +-- Tunny/Solver/Hypervolume.cs | 152 ------------------------------------ 3 files changed, 6 insertions(+), 158 deletions(-) delete mode 100644 Tunny/Solver/Hypervolume.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d96fe4f..4b6bd049 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,7 @@ for soon-to-be removed features. ### Removed -for now removed features. +- Show hypervolume ratio while optimization running ### Fixed diff --git a/Tunny/Solver/Algorithm.cs b/Tunny/Solver/Algorithm.cs index 2a78bc5d..48cbcc23 100644 --- a/Tunny/Solver/Algorithm.cs +++ b/Tunny/Solver/Algorithm.cs @@ -456,34 +456,34 @@ private bool CheckOptimizeComplete(int nTrials, double timeout, int trialNum, Da private ProgressState SetProgressState(OptimizationHandlingInfo optSet, Parameter[] parameter, int trialNum, DateTime startTime) { TLog.MethodStart(); - ComputeBestValues(optSet.Study, trialNum, out double[][] bestValues, out double hypervolumeRatio); + double[][] bestValues = ComputeBestValues(optSet.Study); return new ProgressState { TrialNumber = trialNum, ObjectiveNum = Objective.Length, BestValues = bestValues, Parameter = parameter, - HypervolumeRatio = hypervolumeRatio, + HypervolumeRatio = 0, EstimatedTimeRemaining = optSet.Timeout <= 0 ? TimeSpan.FromSeconds((DateTime.Now - startTime).TotalSeconds * (optSet.NTrials - trialNum) / (trialNum + 1)) : TimeSpan.FromSeconds(optSet.Timeout - (DateTime.Now - startTime).TotalSeconds) }; } - private void ComputeBestValues(dynamic study, int trialNum, out double[][] bestValues, out double hypervolumeRatio) + private double[][] ComputeBestValues(dynamic study) { TLog.MethodStart(); + double[][] bestValues = Array.Empty(); if (Settings.Optimize.ShowRealtimeResult) { dynamic[] bestTrials = study.best_trials; bestValues = bestTrials.Select(t => (double[])t.values).ToArray(); - hypervolumeRatio = trialNum == 0 ? 0 : trialNum == 1 || Objective.Length == 1 ? 1 : Hypervolume.Compute2dHypervolumeRatio(study); } else { bestValues = null; - hypervolumeRatio = 0; } + return bestValues; } private void SaveInMemoryStudy(dynamic storage) diff --git a/Tunny/Solver/Hypervolume.cs b/Tunny/Solver/Hypervolume.cs deleted file mode 100644 index 11158048..00000000 --- a/Tunny/Solver/Hypervolume.cs +++ /dev/null @@ -1,152 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; - -using Python.Runtime; - -using Tunny.Core.Settings; -using Tunny.Core.Util; - -namespace Tunny.Solver -{ - public static class Hypervolume - { - public static dynamic CreateFigure(dynamic study, Plot pSettings) - { - TLog.MethodStart(); - var trials = (dynamic[])study.trials; - int[] objIndex = pSettings.TargetObjectiveIndex; - - var trialValues = new List(); - foreach (dynamic trial in trials) - { - if (trial.state.ToString() == "TrialState.COMPLETE") - { - double[] values = (double[])trial.values; - double[] targetValues = new double[objIndex.Length]; - for (int i = 0; i < objIndex.Length; i++) - { - targetValues[i] = values[objIndex[i]]; - } - trialValues.Add(targetValues); - } - } - - double[] maxObjValues = new double[objIndex.Length]; - for (int i = 0; i < objIndex.Length; i++) - { - maxObjValues[i] = trialValues.Select(v => v[i]).Max(); - } - - PyList hvs = ComputeHypervolume(trialValues, maxObjValues, out PyList trialNumbers); - return CreateHypervolumeFigure(trials.Length, hvs, trialNumbers); - } - - private static PyList ComputeHypervolume(List trialValues, double[] maxObjValues, out PyList trialNumbers) - { - TLog.MethodStart(); - dynamic optuna = Py.Import("optuna"); - dynamic np = Py.Import("numpy"); - - var hvs = new PyList(); - var rpObj = new PyList(); - trialNumbers = new PyList(); - - foreach (double max in maxObjValues) - { - rpObj.Append(new PyFloat(max)); - } - dynamic referencePoint = np.array(rpObj); - - dynamic wfg = optuna._hypervolume.WFG(); - for (int i = 1; i < trialValues.Count + 1; i++) - { - var vector = new PyList(); - for (int j = 0; j < i; j++) - { - vector.Append(trialValues[j].ToPython()); - } - hvs.Append(wfg.compute(np.array(vector), referencePoint)); - trialNumbers.Append(new PyInt(i)); - } - return hvs; - } - - private static dynamic CreateHypervolumeFigure(int trialLength, PyList hvs, PyList trialNumbers) - { - TLog.MethodStart(); - dynamic go = Py.Import("plotly.graph_objects"); - - var plotItems = new PyDict(); - plotItems.SetItem("x", trialNumbers); - plotItems.SetItem("y", hvs); - - var plotRange = new PyDict(); - var rangeObj = new PyObject[] { new PyFloat(0), new PyFloat(trialLength + 1) }; - plotRange.SetItem("range", new PyList(rangeObj)); - - dynamic fig = go.Figure(); - fig.add_trace(go.Scatter(plotItems, name: "Hypervolume")); - fig.update_layout(xaxis: plotRange); - fig.update_xaxes(title_text: "#Trials"); - fig.update_yaxes(title_text: "Hypervolume"); - - return fig; - } - - public static double Compute2dHypervolumeRatio(dynamic study) - { - TLog.MethodStart(); - var trials = (dynamic[])study.trials; - var trialValues = new List(); - for (int i = 0; i < trials.Length - 1; i++) - { - dynamic trial = trials[i]; - trialValues.Add((double[])trial.values); - } - double[] maxObjectiveValues = new double[2]; - for (int i = 0; i < 2; i++) - { - maxObjectiveValues[i] = trialValues.Select(v => v[i]).Max(); - } - - return ComputeRatio(trials, trialValues.Count - 1, trialValues.Count, maxObjectiveValues); - } - - private static double ComputeRatio(dynamic[] trials, int baseIndex, int targetIndex, double[] maxObjectiveValues) - { - TLog.MethodStart(); - dynamic np = Py.Import("numpy"); - var rpObj = new PyList(); - - foreach (double max in maxObjectiveValues) - { - rpObj.Append(new PyFloat(max)); - } - dynamic referencePoint = np.array(rpObj); - - double baseHypervolume = Wfg(trials, baseIndex, referencePoint); - double targetHypervolume = Wfg(trials, targetIndex, referencePoint); - - return Math.Round(targetHypervolume / baseHypervolume, 3); - } - - private static double Wfg(dynamic[] trials, int baseIndex, dynamic referencePoint) - { - TLog.MethodStart(); - dynamic optuna = Py.Import("optuna"); - dynamic np = Py.Import("numpy"); - - var vectors = new PyList(); - dynamic wfg = optuna._hypervolume.WFG(); - for (int j = 0; j < baseIndex; j++) - { - var vector = new PyList(); - vector.Append(trials[j].values[0]); - vector.Append(trials[j].values[1]); - vectors.Append(vector); - } - return (double)wfg.compute(np.array(vectors), referencePoint); - } - } -} From dead5e020a79c629385b15188e91749f6a14c04f Mon Sep 17 00:00:00 2001 From: hrntsm Date: Sat, 24 Feb 2024 13:01:05 +0900 Subject: [PATCH 3/4] Add LoadStudy method to Study class and use it in Visualize class --- Optuna/Study/Study.cs | 5 +++++ Tunny/Solver/Visualize.cs | 17 ++--------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/Optuna/Study/Study.cs b/Optuna/Study/Study.cs index 60f47b78..c65dc5e4 100644 --- a/Optuna/Study/Study.cs +++ b/Optuna/Study/Study.cs @@ -146,5 +146,10 @@ public static dynamic CreateStudy(dynamic optuna, string studyName, dynamic samp load_if_exists: loadIfExists ); } + + public static dynamic LoadStudy(dynamic optuna, dynamic storage, string studyName) + { + return optuna.load_study(storage: storage, study_name: studyName); + } } } diff --git a/Tunny/Solver/Visualize.cs b/Tunny/Solver/Visualize.cs index 2205b81b..d6cac460 100644 --- a/Tunny/Solver/Visualize.cs +++ b/Tunny/Solver/Visualize.cs @@ -1,6 +1,7 @@ using System; using System.Windows.Forms; +using Optuna.Study; using Optuna.Visualization; using Python.Runtime; @@ -24,20 +25,6 @@ public Visualize(TSettings settings, bool hasConstraint) _hasConstraint = hasConstraint; } - private static dynamic LoadStudy(dynamic optuna, dynamic storage, string studyName) - { - TLog.MethodStart(); - try - { - return optuna.load_study(storage: storage, study_name: studyName); - } - catch (Exception e) - { - TunnyMessageBox.Show(e.Message, "Tunny", MessageBoxButtons.OK, MessageBoxIcon.Error); - return null; - } - } - public void Plot(Plot pSettings) { TLog.MethodStart(); @@ -46,7 +33,7 @@ public void Plot(Plot pSettings) { dynamic optuna = Py.Import("optuna"); dynamic storage = _settings.Storage.CreateNewOptunaStorage(false); - dynamic study = LoadStudy(optuna, storage, pSettings.TargetStudyName); + dynamic study = Study.LoadStudy(optuna, storage, pSettings.TargetStudyName); if (study == null) { return; From 985cf1ca3649e7510829b906f6cdc5e0817d292c Mon Sep 17 00:00:00 2001 From: hrntsm Date: Sat, 24 Feb 2024 13:09:11 +0900 Subject: [PATCH 4/4] Clean code --- Tunny/Solver/Algorithm.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Tunny/Solver/Algorithm.cs b/Tunny/Solver/Algorithm.cs index 48cbcc23..e99ec10c 100644 --- a/Tunny/Solver/Algorithm.cs +++ b/Tunny/Solver/Algorithm.cs @@ -473,17 +473,15 @@ private ProgressState SetProgressState(OptimizationHandlingInfo optSet, Paramete private double[][] ComputeBestValues(dynamic study) { TLog.MethodStart(); - double[][] bestValues = Array.Empty(); if (Settings.Optimize.ShowRealtimeResult) { dynamic[] bestTrials = study.best_trials; - bestValues = bestTrials.Select(t => (double[])t.values).ToArray(); + return bestTrials.Select(t => (double[])t.values).ToArray(); } else { - bestValues = null; + return null; } - return bestValues; } private void SaveInMemoryStudy(dynamic storage)