Merge pull request #162 from hrntsm/fix/storage-handling

Fix storage handling

CHANGELOG.md

@@ -6,6 +6,22 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p
 
 Please see [here](https://github.com/hrntsm/Tunny/releases) for the data released for each version.
 
+## [UNRELEASED] -xxxx-xx-xx
+
+### Added
+
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+- bug where visualize and output did not work when using journal storage.
+
+### Security
+
 ## [v0.7.0] -2023-03-21
 
 ### Added
@@ -24,10 +40,6 @@ Please see [here](https://github.com/hrntsm/Tunny/releases) for the data release
   - If you want to install it again, you can do so by checking the checkbox from Misc in the Settings tab.
 - The most of the dll files are combined into a single gha file to improve usability.
 
-### Deprecated
-
-### Removed
-
 ### Fixed
 
 - Enabled Optuna-Dashboard to work even if the filename contains spaces.
@@ -70,8 +82,6 @@ Please see [here](https://github.com/hrntsm/Tunny/releases) for the data release
 
 - Disable unused setting tab ui items
 
-### Removed
-
 ### Fixed
 
 - Even if there was an error in the input to the Tunny component, a window could be launched and the button to perform optimization could be pressed, so we made sure that this would not happen.

Tunny/Settings/Storage.cs

@@ -1,9 +1,68 @@
-namespace Tunny.Settings
+using System;
+
+using Tunny.Storage;
+
+namespace Tunny.Settings
 {
     public class Storage
     {
         public string Path { get; set; } = "/fish.log";
         public StorageType Type { get; set; } = StorageType.Journal;
+
+        public string GetOptunaStoragePath()
+        {
+            switch (Type)
+            {
+                case StorageType.InMemory:
+                    return string.Empty;
+                case StorageType.Sqlite:
+                    return "sqlite:///" + Path;
+                case StorageType.Postgres:
+                case StorageType.MySql:
+                    throw new NotImplementedException();
+                case StorageType.Journal:
+                    return Path;
+                default:
+                    throw new NotImplementedException();
+            }
+        }
+
+        public string GetOptunaStoragePathByExtension()
+        {
+            switch (System.IO.Path.GetExtension(Path))
+            {
+                case null:
+                    return string.Empty;
+                case ".sqlite3":
+                case ".db":
+                    return @"sqlite:///" + $"\"{Path}\"";
+                case ".log":
+                    return $"\"{Path}\"";
+                default:
+                    throw new NotImplementedException();
+            }
+        }
+
+        public dynamic CreateNewOptunaStorage(bool useInnerPythonEngine)
+        {
+            dynamic storage;
+            switch (Type)
+            {
+                case StorageType.InMemory:
+                    storage = new InMemoryStorage().CreateNewStorage(useInnerPythonEngine, this);
+                    break;
+                case StorageType.Sqlite:
+                    storage = new SqliteStorage().CreateNewStorage(useInnerPythonEngine, this);
+                    break;
+                case StorageType.Journal:
+                    storage = new JournalStorage().CreateNewStorage(useInnerPythonEngine, this);
+                    break;
+                default:
+                    throw new ArgumentException("Storage type is not defined.");
+            }
+
+            return storage;
+        }
     }
 
     public enum StorageType

Tunny/Solver/Algorithm.cs

@@ -56,7 +56,7 @@ public void Solve()
             {
                 dynamic optuna = Py.Import("optuna");
                 dynamic sampler = SetSamplerSettings(samplerType, ref nTrials, optuna, HasConstraints);
-                dynamic storage = CreateNewStorage();
+                dynamic storage = Settings.Storage.CreateNewOptunaStorage(false);
 
                 if (CheckExistStudyParameter(nObjective, optuna, storage))
                 {
@@ -69,27 +69,6 @@ public void Solve()
             PythonEngine.Shutdown();
         }
 
-        private dynamic CreateNewStorage()
-        {
-            dynamic storage;
-            switch (Settings.Storage.Type)
-            {
-                case StorageType.InMemory:
-                    storage = new InMemoryStorage().CreateNewStorage(false, string.Empty);
-                    break;
-                case StorageType.Sqlite:
-                    storage = new SqliteStorage().CreateNewStorage(false, Settings.Storage.Path);
-                    break;
-                case StorageType.Journal:
-                    storage = new JournalStorage().CreateNewStorage(false, Settings.Storage.Path);
-                    break;
-                default:
-                    throw new ArgumentException("Storage type is not defined.");
-            }
-
-            return storage;
-        }
-
         private static StringBuilder NicknameToAttr(IEnumerable<string> nicknames)
         {
             var name = new StringBuilder();
@@ -305,7 +284,7 @@ private void CopyInMemoryStudy(dynamic storage)
         {
             dynamic optuna = Py.Import("optuna");
             string studyName = Settings.StudyName;
-            optuna.copy_study(from_study_name: studyName, to_study_name: studyName, from_storage: storage, to_storage: new StorageHandler().CreateNewStorage(false, Settings.Storage.Path));
+            optuna.copy_study(from_study_name: studyName, to_study_name: studyName, from_storage: storage, to_storage: new StorageHandler().CreateNewStorage(false, Settings.Storage));
         }
 
         private static dynamic EnqueueTrial(dynamic study, Dictionary<string, FishEgg> enqueueItems)

Tunny/Solver/Optuna.cs

@@ -97,7 +97,6 @@ private static void ShowErrorMessages(Exception e)
 
         public ModelResult[] GetModelResult(int[] resultNum, string studyName, BackgroundWorker worker)
         {
-            string storage = "sqlite:///" + _settings.Storage.Path;
             var modelResult = new List<ModelResult>();
             PythonEngine.Initialize();
             using (Py.GIL())
@@ -107,6 +106,7 @@ public ModelResult[] GetModelResult(int[] resultNum, string studyName, Backgroun
 
                 try
                 {
+                    dynamic storage = _settings.Storage.CreateNewOptunaStorage(false);
                     study = optuna.load_study(storage: storage, study_name: studyName);
                 }
                 catch (Exception e)

Tunny/Solver/Sampler.cs

@@ -31,7 +31,7 @@ internal static dynamic CmaEs(dynamic optuna, TunnySettings settings)
                     restart_strategy: cmaEs.RestartStrategy == string.Empty ? null : cmaEs.RestartStrategy,
                     inc_popsize: cmaEs.IncPopsize,
                     popsize: cmaEs.PopulationSize,
-                    source_trials: optuna.load_study(study_name: cmaEs.WarmStartStudyName, storage: "sqlite:///" + settings.Storage.Path).get_trials()
+                    source_trials: optuna.load_study(study_name: cmaEs.WarmStartStudyName, storage: settings.Storage.GetOptunaStoragePath()).get_trials()
                 )
                 : optuna.samplers.CmaEsSampler(
                     sigma0: cmaEs.Sigma0,

Tunny/Solver/Visualize.cs

@@ -20,7 +20,7 @@ public Visualize(TunnySettings settings, bool hasConstraint)
             _hasConstraint = hasConstraint;
         }
 
-        private static dynamic LoadStudy(dynamic optuna, string storage, string studyName)
+        private static dynamic LoadStudy(dynamic optuna, dynamic storage, string studyName)
         {
             try
             {
@@ -35,11 +35,11 @@ private static dynamic LoadStudy(dynamic optuna, string storage, string studyNam
 
         public void Plot(PlotSettings pSettings)
         {
-            string storage = "sqlite:///" + _settings.Storage.Path;
             PythonEngine.Initialize();
             using (Py.GIL())
             {
                 dynamic optuna = Py.Import("optuna");
+                dynamic storage = _settings.Storage.CreateNewOptunaStorage(false);
                 dynamic study = LoadStudy(optuna, storage, pSettings.TargetStudyName);
                 if (study == null)
                 {
@@ -274,11 +274,11 @@ private static void SaveFigure(dynamic fig, string name)
 
         public void ClusteringPlot(PlotSettings pSettings)
         {
-            string storage = "sqlite:///" + _settings.Storage.Path;
             PythonEngine.Initialize();
             using (Py.GIL())
             {
                 dynamic optuna = Py.Import("optuna");
+                dynamic storage = _settings.Storage.CreateNewOptunaStorage(false);
                 dynamic study = LoadStudy(optuna, storage, pSettings.TargetStudyName);
                 if (study == null)
                 {

Tunny/Storage/IStorage.cs

@@ -2,13 +2,13 @@ namespace Tunny.Storage
 {
     public interface IStorage : ICreateStorage
     {
-        void DuplicateStudyInStorage(string fromStudyName, string toStudyName, string storagePath);
+        void DuplicateStudyInStorage(string fromStudyName, string toStudyName, Settings.Storage storageSetting);
         StudySummary[] GetStudySummaries(string storagePath);
     }
 
     public interface ICreateStorage
     {
         dynamic Storage { get; set; }
-        dynamic CreateNewStorage(bool useInnerPythonEngine, string storagePath);
+        dynamic CreateNewStorage(bool useInnerPythonEngine, Settings.Storage storageSetting);
     }
 }

Tunny/Storage/InMemoryStorage.cs

@@ -8,7 +8,7 @@ public class InMemoryStorage : PythonInit, ICreateStorage
     {
         public dynamic Storage { get; set; }
 
-        public dynamic CreateNewStorage(bool useInnerPythonEngine, string storagePath)
+        public dynamic CreateNewStorage(bool useInnerPythonEngine, Settings.Storage storageSetting)
         {
             if (useInnerPythonEngine)
             {

Tunny/Storage/JournalStorage.cs

@@ -157,8 +157,9 @@ private static void SetStudySummaryValue(IGrouping<int?, JournalStorage> group,
             }
         }
 
-        public dynamic CreateNewStorage(bool useInnerPythonEngine, string storagePath)
+        public dynamic CreateNewStorage(bool useInnerPythonEngine, Settings.Storage storageSetting)
         {
+            string storagePath = storageSetting.GetOptunaStoragePath();
             if (useInnerPythonEngine)
             {
                 PythonEngine.Initialize();
@@ -184,13 +185,13 @@ private void CreateStorageProcess(string storagePath)
             Storage = optuna.storages.JournalStorage(optuna.storages.JournalFileStorage(filePath, lock_obj: lockObj));
         }
 
-        public void DuplicateStudyInStorage(string fromStudyName, string toStudyName, string storagePath)
+        public void DuplicateStudyInStorage(string fromStudyName, string toStudyName, Settings.Storage storageSetting)
         {
             PythonEngine.Initialize();
             using (Py.GIL())
             {
                 dynamic optuna = Py.Import("optuna");
-                dynamic storage = CreateNewStorage(false, storagePath);
+                dynamic storage = CreateNewStorage(false, storageSetting);
                 optuna.copy_study(from_study_name: fromStudyName, to_study_name: toStudyName, from_storage: storage, to_storage: storage);
             }
             PythonEngine.Shutdown();

Tunny/Storage/SqliteStorage.cs

@@ -107,32 +107,9 @@ private static void GetStudy(List<StudySummary> studySummaries, SQLiteConnection
             }
         }
 
-        public static StudySummary[] GetStudySummariesPY(string storagePath)
+        public dynamic CreateNewStorage(bool useInnerPythonEngine, Settings.Storage storageSetting)
         {
-            var studySummaries = new List<StudySummary>();
-            string storage = "sqlite:///" + storagePath;
-            PythonEngine.Initialize();
-            using (Py.GIL())
-            {
-                dynamic optuna = Py.Import("optuna");
-                PyList summaries = optuna.study.get_all_study_summaries(storage);
-                foreach (dynamic summary in summaries)
-                {
-                    var studySummary = new StudySummary
-                    {
-                        StudyName = (string)summary.study_name,
-                        NTrials = (int)summary.n_trials,
-                    };
-                    studySummaries.Add(studySummary);
-                }
-            }
-            PythonEngine.Shutdown();
-            return studySummaries.ToArray();
-        }
-
-        public dynamic CreateNewStorage(bool useInnerPythonEngine, string storagePath)
-        {
-            string sqlitePath = "sqlite:///" + storagePath;
+            string sqlitePath = storageSetting.GetOptunaStoragePath();
             if (useInnerPythonEngine)
             {
                 PythonEngine.Initialize();
@@ -156,9 +133,9 @@ private void CreateStorageProcess(string sqlitePath)
             Storage = optuna.storages.RDBStorage(sqlitePath);
         }
 
-        public void DuplicateStudyInStorage(string fromStudyName, string toStudyName, string storagePath)
+        public void DuplicateStudyInStorage(string fromStudyName, string toStudyName, Settings.Storage storageSetting)
         {
-            string storage = "sqlite:///" + storagePath;
+            string storage = storageSetting.GetOptunaStoragePath();
             PythonEngine.Initialize();
             using (Py.GIL())
             {

Tunny/Storage/StorageHandler.cs

@@ -6,11 +6,11 @@ public class StorageHandler : IStorage
     {
         public dynamic Storage { get; set; }
 
-        public dynamic CreateNewStorage(bool useInnerPythonEngine, string storagePath)
+        public dynamic CreateNewStorage(bool useInnerPythonEngine, Settings.Storage storageSetting)
         {
             ICreateStorage storage;
 
-            switch (Path.GetExtension(storagePath))
+            switch (Path.GetExtension(storageSetting.Path))
             {
                 case null:
                     storage = new InMemoryStorage();
@@ -25,15 +25,15 @@ public dynamic CreateNewStorage(bool useInnerPythonEngine, string storagePath)
                 default:
                     throw new ArgumentException("Storage type not supported");
             }
-            Storage = storage.CreateNewStorage(useInnerPythonEngine, storagePath);
+            Storage = storage.CreateNewStorage(useInnerPythonEngine, storageSetting);
             return Storage;
         }
 
-        public void DuplicateStudyInStorage(string fromStudyName, string toStudyName, string storagePath)
+        public void DuplicateStudyInStorage(string fromStudyName, string toStudyName, Settings.Storage storageSetting)
         {
             IStorage storage;
 
-            switch (Path.GetExtension(storagePath))
+            switch (Path.GetExtension(storageSetting.Path))
             {
                 case ".sqlite3":
                 case ".db":
@@ -45,7 +45,7 @@ public void DuplicateStudyInStorage(string fromStudyName, string toStudyName, st
                 default:
                     throw new ArgumentException("Storage type not supported");
             }
-            storage.DuplicateStudyInStorage(fromStudyName, toStudyName, storagePath);
+            storage.DuplicateStudyInStorage(fromStudyName, toStudyName, storageSetting);
         }
 
         public StudySummary[] GetStudySummaries(string storagePath)

Tunny/UI/OptimizeWindowTab/FileTab.cs

@@ -48,7 +48,7 @@ private void SetResultFilePathButton_Click(object sender, EventArgs e)
                 string storagePath = _settings.Storage.Path;
                 if (!File.Exists(storagePath))
                 {
-                    new StorageHandler().CreateNewStorage(true, storagePath);
+                    new StorageHandler().CreateNewStorage(true, _settings.Storage);
                 }
                 UpdateStudyComboBox(storagePath);
             }

Tunny/UI/OptimizeWindowTab/OptimizeTab.cs

@@ -96,7 +96,7 @@ private bool CheckInputValue(GH_DocumentEditor ghCanvas)
             }
             else if (checkResult && copyStudyCheckBox.Enabled && copyStudyCheckBox.Checked)
             {
-                new StorageHandler().DuplicateStudyInStorage(existingStudyComboBox.Text, studyNameTextBox.Text, _settings.Storage.Path);
+                new StorageHandler().DuplicateStudyInStorage(existingStudyComboBox.Text, studyNameTextBox.Text, _settings.Storage);
                 _settings.StudyName = studyNameTextBox.Text;
             }
             else if (checkResult && continueStudyCheckBox.Checked)

Tunny/UI/OptimizeWindowTab/VisualizeTab.cs

@@ -24,9 +24,7 @@ private void DashboardButton_Click(object sender, EventArgs e)
             CheckExistDashboardProcess();
             var dashboard = new Process();
             dashboard.StartInfo.FileName = PythonInstaller.GetEmbeddedPythonPath() + @"\Scripts\optuna-dashboard.exe";
-            dashboard.StartInfo.Arguments = Path.GetExtension(_settings.Storage.Path) == ".log"
-                ? $"\"{_settings.Storage.Path}\""
-                : @"sqlite:///" + $"\"{_settings.Storage.Path}\"";
+            dashboard.StartInfo.Arguments = _settings.Storage.GetOptunaStoragePathByExtension();
             dashboard.StartInfo.UseShellExecute = false;
             dashboard.StartInfo.WindowStyle = ProcessWindowStyle.Minimized;
             dashboard.Start();