From 0c0bee185c4c0f9c216c48dd6cacb6ba60ddef27 Mon Sep 17 00:00:00 2001 From: Michel Kok Date: Mon, 17 Apr 2023 10:50:43 +0200 Subject: [PATCH 1/6] Consistent resuming of studies with Optuna. --- .../hydra_plugins/hydra_optuna_sweeper/_impl.py | 7 +++++-- .../hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py index d17bbb1f70a..653ae46f8e2 100644 --- a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py +++ b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py @@ -159,6 +159,7 @@ def __init__( search_space: Optional[DictConfig], custom_search_space: Optional[str], params: Optional[DictConfig], + load_if_exists: Optional[bool] = True, ) -> None: self.sampler = sampler self.direction = direction @@ -169,6 +170,7 @@ def __init__( self.max_failure_rate = max_failure_rate assert self.max_failure_rate >= 0.0 assert self.max_failure_rate <= 1.0 + self.load_if_exists = load_if_exists self.custom_search_space_extender: Optional[ Callable[[DictConfig, Trial], None] ] = None @@ -330,7 +332,7 @@ def sweep(self, arguments: List[str]) -> None: storage=self.storage, sampler=self.sampler, directions=directions, - load_if_exists=True, + load_if_exists=self.load_if_exists, ) log.info(f"Study name: {study.study_name}") log.info(f"Storage: {self.storage}") @@ -338,7 +340,8 @@ def sweep(self, arguments: List[str]) -> None: log.info(f"Directions: {directions}") batch_size = self.n_jobs - n_trials_to_go = self.n_trials + n_trials_to_go = self.n_trials - len(study.trials) + self.job_idx = len(study.trials) while n_trials_to_go > 0: batch_size = min(n_trials_to_go, batch_size) diff --git a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py index c53d5d7b5d0..6ffb786c844 100644 --- a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py +++ b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py @@ -23,6 +23,7 @@ def __init__( search_space: Optional[DictConfig], custom_search_space: Optional[str], params: Optional[DictConfig], + load_if_exists: Optional[bool] = True, ) -> None: from ._impl import OptunaSweeperImpl @@ -37,6 +38,7 @@ def __init__( search_space, custom_search_space, params, + load_if_exists, ) def setup( From 1f98e7048996bada95824b00df5bb067f6857502 Mon Sep 17 00:00:00 2001 From: Michel Kok Date: Mon, 17 Apr 2023 11:00:40 +0200 Subject: [PATCH 2/6] Update docs. --- plugins/hydra_optuna_sweeper/example/conf/config.yaml | 1 + website/docs/plugins/optuna_sweeper.md | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/hydra_optuna_sweeper/example/conf/config.yaml b/plugins/hydra_optuna_sweeper/example/conf/config.yaml index e3b86c71042..68bce7a5f8f 100644 --- a/plugins/hydra_optuna_sweeper/example/conf/config.yaml +++ b/plugins/hydra_optuna_sweeper/example/conf/config.yaml @@ -12,6 +12,7 @@ hydra: n_trials: 20 n_jobs: 1 max_failure_rate: 0.0 + load_if_exists: true params: x: range(-5.5, 5.5, step=0.5) y: choice(-5 ,0 ,5) diff --git a/website/docs/plugins/optuna_sweeper.md b/website/docs/plugins/optuna_sweeper.md index b8364809679..9f603f713b2 100644 --- a/website/docs/plugins/optuna_sweeper.md +++ b/website/docs/plugins/optuna_sweeper.md @@ -71,6 +71,7 @@ study_name: sphere n_trials: 20 n_jobs: 1 max_failure_rate: 0.0 +load_if_exists: true params: x: range(-5.5,5.5,step=0.5) y: choice(-5,0,5) From e22d6da1c07829df58c677cf4059c9f5d4c87b38 Mon Sep 17 00:00:00 2001 From: Michel Kok Date: Fri, 21 Apr 2023 17:07:30 +0200 Subject: [PATCH 3/6] Define tests accordingly. --- .../hydra_optuna_sweeper/_impl.py | 2 +- .../hydra_optuna_sweeper/config.py | 3 ++ .../hydra_optuna_sweeper/optuna_sweeper.py | 4 +- .../tests/test_optuna_sweeper_plugin.py | 42 +++++++++++++++++++ 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py index 653ae46f8e2..b501f29021a 100644 --- a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py +++ b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py @@ -156,10 +156,10 @@ def __init__( n_trials: int, n_jobs: int, max_failure_rate: float, + load_if_exists: Optional[bool], search_space: Optional[DictConfig], custom_search_space: Optional[str], params: Optional[DictConfig], - load_if_exists: Optional[bool] = True, ) -> None: self.sampler = sampler self.direction = direction diff --git a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/config.py b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/config.py index 03fde5a12d0..5bd97633b56 100644 --- a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/config.py +++ b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/config.py @@ -175,6 +175,9 @@ class OptunaSweeperConf: # Maximum authorized failure rate for a batch of parameters max_failure_rate: float = 0.0 + # Load an existing study and resume it. + load_if_exists: bool = True + search_space: Optional[Dict[str, Any]] = None params: Optional[Dict[str, str]] = None diff --git a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py index 6ffb786c844..fd09c1f249c 100644 --- a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py +++ b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py @@ -20,10 +20,10 @@ def __init__( n_trials: int, n_jobs: int, max_failure_rate: float, + load_if_exists: Optional[bool], search_space: Optional[DictConfig], custom_search_space: Optional[str], params: Optional[DictConfig], - load_if_exists: Optional[bool] = True, ) -> None: from ._impl import OptunaSweeperImpl @@ -35,10 +35,10 @@ def __init__( n_trials, n_jobs, max_failure_rate, + load_if_exists, search_space, custom_search_space, params, - load_if_exists, ) def setup( diff --git a/plugins/hydra_optuna_sweeper/tests/test_optuna_sweeper_plugin.py b/plugins/hydra_optuna_sweeper/tests/test_optuna_sweeper_plugin.py index f042937a8fd..252b7036900 100644 --- a/plugins/hydra_optuna_sweeper/tests/test_optuna_sweeper_plugin.py +++ b/plugins/hydra_optuna_sweeper/tests/test_optuna_sweeper_plugin.py @@ -6,6 +6,7 @@ from typing import Any, List, Optional import optuna + from hydra.core.override_parser.overrides_parser import OverridesParser from hydra.core.plugins import Plugins from hydra.plugins.sweeper import Sweeper @@ -331,6 +332,7 @@ def test_warnings( n_trials=1, n_jobs=1, max_failure_rate=0.0, + load_if_exists=True, custom_search_space=None, ) if search_space is not None: @@ -371,6 +373,46 @@ def test_failure_rate(max_failure_rate: float, tmpdir: Path) -> None: assert error_string not in err +@mark.parametrize("load_if_exists", (True, False, None)) +def test_load_if_exists(load_if_exists: Optional[bool], tmpdir: Path) -> None: + storage = "sqlite:///" + os.path.join(str(tmpdir), "test.db") + study_name = "test-optuna-example" + + # Let the first run fail. + cmd = [ + sys.executable, + "example/sphere.py", + "--multirun", + "hydra.sweep.dir=" + str(tmpdir), + "hydra.job.chdir=True", + "hydra.sweeper.n_trials=15", + "hydra.sweeper.n_jobs=1", + f"hydra.sweeper.storage={storage}", + f"hydra.sweeper.study_name={study_name}", + "hydra/sweeper/sampler=random", + "hydra.sweeper.sampler.seed=123", + ] + _ = run_process(cmd, print_error=False, raise_exception=False) + + cmd.pop(-1) + if load_if_exists is True: + cmd.append("hydra.sweeper.load_if_exists=True") + + elif load_if_exists is False: + cmd.append("hydra.sweeper.load_if_exists=False") + + _, err = run_process(cmd, print_error=False, raise_exception=False) + + error_string = ( + "optuna.exceptions.DuplicatedStudyError: " + "Another study with name 'test-optuna-example' already exists." + ) + if load_if_exists is False: + assert error_string in err + else: + assert error_string not in err + + def test_example_with_deprecated_search_space( tmpdir: Path, ) -> None: From cfaabc339d55c84d319a18f9bbba465feaa85229 Mon Sep 17 00:00:00 2001 From: Michel Kok Date: Fri, 21 Apr 2023 17:14:07 +0200 Subject: [PATCH 4/6] Check spelling. --- plugins/hydra_optuna_sweeper/example/conf/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hydra_optuna_sweeper/example/conf/config.yaml b/plugins/hydra_optuna_sweeper/example/conf/config.yaml index 68bce7a5f8f..b80dae6ad84 100644 --- a/plugins/hydra_optuna_sweeper/example/conf/config.yaml +++ b/plugins/hydra_optuna_sweeper/example/conf/config.yaml @@ -12,7 +12,7 @@ hydra: n_trials: 20 n_jobs: 1 max_failure_rate: 0.0 - load_if_exists: true + load_if_exists: True params: x: range(-5.5, 5.5, step=0.5) y: choice(-5 ,0 ,5) From 91fb8f67f3636367116cdb3d648aa69e5dd875d0 Mon Sep 17 00:00:00 2001 From: Michel Kok Date: Fri, 21 Apr 2023 17:18:20 +0200 Subject: [PATCH 5/6] Check spelling. --- website/docs/plugins/optuna_sweeper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/plugins/optuna_sweeper.md b/website/docs/plugins/optuna_sweeper.md index 9f603f713b2..83ecbabafb7 100644 --- a/website/docs/plugins/optuna_sweeper.md +++ b/website/docs/plugins/optuna_sweeper.md @@ -71,7 +71,7 @@ study_name: sphere n_trials: 20 n_jobs: 1 max_failure_rate: 0.0 -load_if_exists: true +load_if_exists: True params: x: range(-5.5,5.5,step=0.5) y: choice(-5,0,5) From 7dbf1f885502ba81415c811357db6a0ecdaf168f Mon Sep 17 00:00:00 2001 From: Michel Kok Date: Mon, 21 Aug 2023 14:31:03 +0200 Subject: [PATCH 6/6] Ignore new deprecation message from setuptools. #2731 --- pytest.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index bfbef0b9979..04a5d665968 100644 --- a/pytest.ini +++ b/pytest.ini @@ -15,4 +15,6 @@ filterwarnings = ; Remove when default changes ignore:.*Future Hydra versions will no longer change working directory at job runtime by default.*:UserWarning ; Jupyter notebook test on Windows yield warnings - ignore:.*Proactor event loop does not implement add_reader family of methods required for zmq.*:RuntimeWarning \ No newline at end of file + ignore:.*Proactor event loop does not implement add_reader family of methods required for zmq.*:RuntimeWarning + ; setuptools 67.5.0+ emits this warning when setuptools.commands.develop is imported + ignore:pkg_resources is deprecated as an API:DeprecationWarning \ No newline at end of file