From 215cb2d4047cc6ac3972ab5e1c53727d47ddb86c Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 17 Jul 2024 10:09:45 +0200 Subject: [PATCH 1/9] Upgrad to ConfigSpace 1.x --- CHANGELOG.md | 3 ++- setup.py | 2 +- smac/acquisition/maximizer/local_search.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b209815d3..045253993 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ - Add example to specify total budget (fidelity units) instead of n_trials for multi-fidelity/Hyperband (#1121) ## Dependencies -- Update numpy NaN (#1122) and restrict numpy and ConfigSpace versions +- Update numpy NaN (#1122) and restrict numpy version +- Upgrade to ConfigSpace 1.x.x (#1124) # 2.1.0 diff --git a/setup.py b/setup.py index 9e8467e4b..970b42604 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def read_file(filepath: str) -> str: "scipy>=1.9.2", "psutil", "pynisher>=1.0.0", - "ConfigSpace>=0.6.1,<1.0.0", + "ConfigSpace>=1.0.0", "joblib", "scikit-learn>=1.1.2", "pyrfr>=0.9.0", diff --git a/smac/acquisition/maximizer/local_search.py b/smac/acquisition/maximizer/local_search.py index 3ef1ae96e..297c032a2 100644 --- a/smac/acquisition/maximizer/local_search.py +++ b/smac/acquisition/maximizer/local_search.py @@ -389,7 +389,7 @@ def _search( if acq_val[acq_index] > acq_val_candidates[i]: is_valid = False try: - neighbors[acq_index].is_valid_configuration() + neighbors[acq_index].check_valid_configuration() is_valid = True except (ValueError, ForbiddenValueError) as e: logger.debug("Local search %d: %s", i, e) From 8864a95d11d0ca5ddc745c08cf00b2e9d77d3320 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 17 Jul 2024 10:27:20 +0200 Subject: [PATCH 2/9] refactor: ConfigSpace API --- benchmark/src/models/ac_branin.py | 2 +- benchmark/src/models/branin.py | 2 +- benchmark/src/models/himmelblau.py | 2 +- benchmark/src/models/mlp.py | 4 ++-- benchmark/src/models/svm.py | 4 ++-- examples/1_basics/1_quadratic_function.py | 2 +- examples/1_basics/2_svm_cv.py | 4 ++-- examples/1_basics/3_ask_and_tell.py | 2 +- examples/1_basics/4_callback.py | 2 +- examples/1_basics/5_continue.py | 2 +- examples/1_basics/6_priors.py | 2 +- examples/1_basics/7_parallelization_cluster.py | 2 +- examples/2_multi_fidelity/1_mlp_epochs.py | 4 ++-- examples/2_multi_fidelity/2_sgd_datasets.py | 2 +- .../2_multi_fidelity/3_specify_HB_via_total_budget.py | 2 +- examples/3_multi_objective/2_parego.py | 4 ++-- examples/4_advanced_optimizer/1_turbo_optimizer.py | 2 +- examples/4_advanced_optimizer/2_boing_optimizer.py | 2 +- examples/4_advanced_optimizer/3_metadata_callback.py | 2 +- .../4_intensify_crossvalidation.py | 2 +- smac/utils/subspaces/__init__.py | 2 +- tests/fixtures/configspace.py | 6 +++--- tests/fixtures/models.py | 4 ++-- tests/test_acquisition/test_maximizers.py | 10 +++++----- tests/test_initial_design/test_sobol_design.py | 2 +- 25 files changed, 37 insertions(+), 37 deletions(-) diff --git a/benchmark/src/models/ac_branin.py b/benchmark/src/models/ac_branin.py index 12287520f..e55c2862a 100644 --- a/benchmark/src/models/ac_branin.py +++ b/benchmark/src/models/ac_branin.py @@ -20,7 +20,7 @@ def configspace(self) -> ConfigurationSpace: x2 = Float("x2", (0, 15), default=7.5) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([x2]) + cs.add([x2]) return cs diff --git a/benchmark/src/models/branin.py b/benchmark/src/models/branin.py index 1fd20554f..0f86d82ac 100644 --- a/benchmark/src/models/branin.py +++ b/benchmark/src/models/branin.py @@ -20,7 +20,7 @@ def configspace(self) -> ConfigurationSpace: x2 = Float("x2", (0, 15), default=0) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([x1, x2]) + cs.add([x1, x2]) return cs diff --git a/benchmark/src/models/himmelblau.py b/benchmark/src/models/himmelblau.py index cab99019a..c12029e1a 100644 --- a/benchmark/src/models/himmelblau.py +++ b/benchmark/src/models/himmelblau.py @@ -19,7 +19,7 @@ def configspace(self) -> ConfigurationSpace: y = Float("y", (-5, 5)) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([x, y]) + cs.add([x, y]) return cs diff --git a/benchmark/src/models/mlp.py b/benchmark/src/models/mlp.py index 4867a2f42..7329de0df 100644 --- a/benchmark/src/models/mlp.py +++ b/benchmark/src/models/mlp.py @@ -33,7 +33,7 @@ def configspace(self) -> ConfigurationSpace: learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) # Add all hyperparameters at once: - cs.add_hyperparameters([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) # Adding conditions to restrict the hyperparameter space... # ... since learning rate is used when solver is 'sgd'. @@ -44,7 +44,7 @@ def configspace(self) -> ConfigurationSpace: use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs diff --git a/benchmark/src/models/svm.py b/benchmark/src/models/svm.py index 88159ab70..3ed294ad0 100644 --- a/benchmark/src/models/svm.py +++ b/benchmark/src/models/svm.py @@ -34,8 +34,8 @@ def configspace(self) -> ConfigurationSpace: use_gamma_value = InCondition(child=gamma_value, parent=gamma, values=["value"]) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([kernel, C, shrinking, degree, coef, gamma, gamma_value]) - cs.add_conditions([use_degree, use_coef, use_gamma, use_gamma_value]) + cs.add([kernel, C, shrinking, degree, coef, gamma, gamma_value]) + cs.add([use_degree, use_coef, use_gamma, use_gamma_value]) return cs diff --git a/examples/1_basics/1_quadratic_function.py b/examples/1_basics/1_quadratic_function.py index 3cafb846b..4d27c7ae6 100644 --- a/examples/1_basics/1_quadratic_function.py +++ b/examples/1_basics/1_quadratic_function.py @@ -26,7 +26,7 @@ class QuadraticFunction: def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x = Float("x", (-5, 5), default=-5) - cs.add_hyperparameters([x]) + cs.add([x]) return cs diff --git a/examples/1_basics/2_svm_cv.py b/examples/1_basics/2_svm_cv.py index cfe39d4bb..345fcffb0 100644 --- a/examples/1_basics/2_svm_cv.py +++ b/examples/1_basics/2_svm_cv.py @@ -46,8 +46,8 @@ def configspace(self) -> ConfigurationSpace: use_gamma_value = InCondition(child=gamma_value, parent=gamma, values=["value"]) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([kernel, C, shrinking, degree, coef, gamma, gamma_value]) - cs.add_conditions([use_degree, use_coef, use_gamma, use_gamma_value]) + cs.add([kernel, C, shrinking, degree, coef, gamma, gamma_value]) + cs.add([use_degree, use_coef, use_gamma, use_gamma_value]) return cs diff --git a/examples/1_basics/3_ask_and_tell.py b/examples/1_basics/3_ask_and_tell.py index 5d0e5e78c..6ab8b5ba8 100644 --- a/examples/1_basics/3_ask_and_tell.py +++ b/examples/1_basics/3_ask_and_tell.py @@ -20,7 +20,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs diff --git a/examples/1_basics/4_callback.py b/examples/1_basics/4_callback.py index c3d66a4b9..0fd9e9d9d 100644 --- a/examples/1_basics/4_callback.py +++ b/examples/1_basics/4_callback.py @@ -27,7 +27,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs diff --git a/examples/1_basics/5_continue.py b/examples/1_basics/5_continue.py index 025856fee..63cfb3957 100644 --- a/examples/1_basics/5_continue.py +++ b/examples/1_basics/5_continue.py @@ -47,7 +47,7 @@ class QuadraticFunction: def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x = Float("x", (-5, 5), default=-5) - cs.add_hyperparameters([x]) + cs.add([x]) return cs diff --git a/examples/1_basics/6_priors.py b/examples/1_basics/6_priors.py index 218bd8f46..49b9aea26 100644 --- a/examples/1_basics/6_priors.py +++ b/examples/1_basics/6_priors.py @@ -101,7 +101,7 @@ def configspace(self) -> ConfigurationSpace: ) # Add all hyperparameters at once: - cs.add_hyperparameters([n_layer, n_neurons, activation, optimizer, batch_size, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, optimizer, batch_size, learning_rate_init]) return cs diff --git a/examples/1_basics/7_parallelization_cluster.py b/examples/1_basics/7_parallelization_cluster.py index 36f79586e..a00bc8319 100644 --- a/examples/1_basics/7_parallelization_cluster.py +++ b/examples/1_basics/7_parallelization_cluster.py @@ -41,7 +41,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-5, log=False) x1 = Float("x1", (0, 15), default=2, log=False) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs diff --git a/examples/2_multi_fidelity/1_mlp_epochs.py b/examples/2_multi_fidelity/1_mlp_epochs.py index 48c027d3b..5cb0aefa0 100644 --- a/examples/2_multi_fidelity/1_mlp_epochs.py +++ b/examples/2_multi_fidelity/1_mlp_epochs.py @@ -65,7 +65,7 @@ def configspace(self) -> ConfigurationSpace: learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) # Add all hyperparameters at once: - cs.add_hyperparameters([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) # Adding conditions to restrict the hyperparameter space... # ... since learning rate is only used when solver is 'sgd'. @@ -76,7 +76,7 @@ def configspace(self) -> ConfigurationSpace: use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs diff --git a/examples/2_multi_fidelity/2_sgd_datasets.py b/examples/2_multi_fidelity/2_sgd_datasets.py index 09864a963..178ea21c2 100644 --- a/examples/2_multi_fidelity/2_sgd_datasets.py +++ b/examples/2_multi_fidelity/2_sgd_datasets.py @@ -76,7 +76,7 @@ def configspace(self) -> ConfigurationSpace: learning_rate = Categorical("learning_rate", ["constant", "invscaling", "adaptive"], default="constant") eta0 = Float("eta0", (0.00001, 1), default=0.1, log=True) # Add the parameters to configuration space - cs.add_hyperparameters([alpha, l1_ratio, learning_rate, eta0]) + cs.add([alpha, l1_ratio, learning_rate, eta0]) return cs diff --git a/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py b/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py index 492cd7f46..7c0ebdcf0 100644 --- a/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py +++ b/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py @@ -31,7 +31,7 @@ class QuadraticFunction: def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x = Float("x", (-5, 5), default=-5) - cs.add_hyperparameters([x]) + cs.add([x]) return cs diff --git a/examples/3_multi_objective/2_parego.py b/examples/3_multi_objective/2_parego.py index 856c2e857..b5294fb98 100644 --- a/examples/3_multi_objective/2_parego.py +++ b/examples/3_multi_objective/2_parego.py @@ -54,14 +54,14 @@ def configspace(self) -> ConfigurationSpace: learning_rate = Categorical("learning_rate", ["constant", "invscaling", "adaptive"], default="constant") learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) - cs.add_hyperparameters([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) use_lr = EqualsCondition(child=learning_rate, parent=solver, value="sgd") use_lr_init = InCondition(child=learning_rate_init, parent=solver, values=["sgd", "adam"]) use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs diff --git a/examples/4_advanced_optimizer/1_turbo_optimizer.py b/examples/4_advanced_optimizer/1_turbo_optimizer.py index 860243c02..dc936f707 100644 --- a/examples/4_advanced_optimizer/1_turbo_optimizer.py +++ b/examples/4_advanced_optimizer/1_turbo_optimizer.py @@ -28,7 +28,7 @@ # cs = ConfigurationSpace(seed=0) # x0 = Float("x0", (-5, 10), default=-3) # x1 = Float("x1", (-5, 10), default=-4) -# cs.add_hyperparameters([x0, x1]) +# cs.add([x0, x1]) # return cs diff --git a/examples/4_advanced_optimizer/2_boing_optimizer.py b/examples/4_advanced_optimizer/2_boing_optimizer.py index 3eb1a1869..815b1f340 100644 --- a/examples/4_advanced_optimizer/2_boing_optimizer.py +++ b/examples/4_advanced_optimizer/2_boing_optimizer.py @@ -27,7 +27,7 @@ # cs = ConfigurationSpace(seed=0) # x0 = Float("x0", (-5, 10), default=-3) # x1 = Float("x1", (-5, 10), default=-4) -# cs.add_hyperparameters([x0, x1]) +# cs.add([x0, x1]) # return cs diff --git a/examples/4_advanced_optimizer/3_metadata_callback.py b/examples/4_advanced_optimizer/3_metadata_callback.py index ab35f2862..b82670dbc 100644 --- a/examples/4_advanced_optimizer/3_metadata_callback.py +++ b/examples/4_advanced_optimizer/3_metadata_callback.py @@ -36,7 +36,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs diff --git a/examples/4_advanced_optimizer/4_intensify_crossvalidation.py b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py index d215dd8ec..222d199f9 100644 --- a/examples/4_advanced_optimizer/4_intensify_crossvalidation.py +++ b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py @@ -39,7 +39,7 @@ def configspace(self) -> ConfigurationSpace: gamma = Float("gamma", (2 ** -15, 2 ** 3), default=1.0, log=True) # Add hyperparameters to our configspace - cs.add_hyperparameters([C, gamma]) + cs.add([C, gamma]) return cs diff --git a/smac/utils/subspaces/__init__.py b/smac/utils/subspaces/__init__.py index 686099830..77d5b97f5 100644 --- a/smac/utils/subspaces/__init__.py +++ b/smac/utils/subspaces/__init__.py @@ -322,7 +322,7 @@ # hp_list.append(hp_new) # # We only consider plain hyperparameters -# self.cs_local.add_hyperparameters(hp_list) +# self.cs_local.add(hp_list) # forbiddens_ss = [] # forbiddens = config_space.get_forbiddens() # for forbidden in forbiddens: diff --git a/tests/fixtures/configspace.py b/tests/fixtures/configspace.py index 77cf9522d..5e6e0ca3f 100644 --- a/tests/fixtures/configspace.py +++ b/tests/fixtures/configspace.py @@ -18,7 +18,7 @@ def configspace_small() -> ConfigurationSpace: c = Categorical("c", ["cat", "dog", "mouse"], default="cat") # Add all hyperparameters at once: - cs.add_hyperparameters([a, b, c]) + cs.add([a, b, c]) return cs @@ -36,7 +36,7 @@ def configspace_large() -> ConfigurationSpace: learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) # Add all hyperparameters at once: - cs.add_hyperparameters( + cs.add( [ n_layer, n_neurons, @@ -57,6 +57,6 @@ def configspace_large() -> ConfigurationSpace: use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs diff --git a/tests/fixtures/models.py b/tests/fixtures/models.py index 29f9bf152..7a7a8fcc0 100644 --- a/tests/fixtures/models.py +++ b/tests/fixtures/models.py @@ -18,7 +18,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs @@ -58,7 +58,7 @@ def configspace(self) -> ConfigurationSpace: eta0 = Float("eta0", (0.00001, 1), default=0.1, log=True) # Add the parameters to configuration space - cs.add_hyperparameters([alpha, l1_ratio, learning_rate, eta0]) + cs.add([alpha, l1_ratio, learning_rate, eta0]) return cs diff --git a/tests/test_acquisition/test_maximizers.py b/tests/test_acquisition/test_maximizers.py index 6d373c237..945dfba7f 100644 --- a/tests/test_acquisition/test_maximizers.py +++ b/tests/test_acquisition/test_maximizers.py @@ -54,8 +54,8 @@ def get_array(self): def configspace_branin() -> ConfigurationSpace: """Returns the branin configspace.""" cs = ConfigurationSpace() - cs.add_hyperparameter(Float("x", (-5, 10))) - cs.add_hyperparameter(Float("y", (0, 15))) + cs.add(Float("x", (-5, 10))) + cs.add(Float("y", (0, 15))) return cs @@ -195,7 +195,7 @@ def configspace() -> ConfigurationSpace: c = Float("c", (0, 1), default=0.5) # Add all hyperparameters at once: - cs.add_hyperparameters([a, b, c]) + cs.add([a, b, c]) return cs @@ -357,7 +357,7 @@ def configspace_rosenbrock(): x2 = UniformIntegerHyperparameter("x2", -5, 5, default_value=5) x3 = CategoricalHyperparameter("x3", [5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4], default_value=5) x4 = UniformIntegerHyperparameter("x4", -5, 5, default_value=5) - uniform_cs.add_hyperparameters([x1, x2, x3, x4]) + uniform_cs.add([x1, x2, x3, x4]) return uniform_cs @@ -373,7 +373,7 @@ def configspace_prior(): "x3", [5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4], default_value=5, weights=[999, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] ) x4 = UniformIntegerHyperparameter("x4", lower=-5, upper=5, default_value=5) - prior_cs.add_hyperparameters([x1, x2, x3, x4]) + prior_cs.add([x1, x2, x3, x4]) return prior_cs diff --git a/tests/test_initial_design/test_sobol_design.py b/tests/test_initial_design/test_sobol_design.py index 8663e2972..8fbb427e8 100644 --- a/tests/test_initial_design/test_sobol_design.py +++ b/tests/test_initial_design/test_sobol_design.py @@ -24,7 +24,7 @@ def test_sobol_design(make_scenario, configspace_large): def test_max_hyperparameters(make_scenario): cs = ConfigurationSpace() hyperparameters = [Float("x%d" % (i + 1), (0, 1)) for i in range(21202)] - cs.add_hyperparameters(hyperparameters) + cs.add(hyperparameters) scenario = make_scenario(cs) From 5cb810221ca612c7443092cbc08b3602f2f6c055 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 17 Jul 2024 14:59:43 +0200 Subject: [PATCH 3/9] fix(jsondump): add numpy encoder ConfigSpace integer hyperparameters do not return python int type but numpy int64, which is not json serializable --- smac/scenario.py | 11 ++++------- smac/utils/numpyencoder.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) create mode 100644 smac/utils/numpyencoder.py diff --git a/smac/scenario.py b/smac/scenario.py index 133ae57b0..ca0df81a2 100644 --- a/smac/scenario.py +++ b/smac/scenario.py @@ -11,9 +11,9 @@ import numpy as np from ConfigSpace import ConfigurationSpace -from ConfigSpace.read_and_write import json as cs_json from smac.utils.logging import get_logger +from smac.utils.numpyencoder import NumpyEncoder logger = get_logger(__name__) @@ -203,12 +203,11 @@ def save(self) -> None: # Save everything filename = self.output_directory / "scenario.json" with open(filename, "w") as fh: - json.dump(data, fh, indent=4) + json.dump(data, fh, indent=4, cls=NumpyEncoder) # Save configspace on its own configspace_filename = self.output_directory / "configspace.json" - with open(configspace_filename, "w") as f: - f.write(cs_json.write(self.configspace)) + self.configspace.to_json(configspace_filename) @staticmethod def load(path: Path) -> Scenario: @@ -224,9 +223,7 @@ def load(path: Path) -> Scenario: # Read configspace configspace_filename = path / "configspace.json" - with open(configspace_filename, "r") as f: - - configspace = cs_json.read(f.read()) + configspace = ConfigurationSpace.from_json(configspace_filename) data["configspace"] = configspace diff --git a/smac/utils/numpyencoder.py b/smac/utils/numpyencoder.py new file mode 100644 index 000000000..ff05d9dbd --- /dev/null +++ b/smac/utils/numpyencoder.py @@ -0,0 +1,30 @@ +from __future__ import annotations +import numpy as np +import json + + +class NumpyEncoder(json.JSONEncoder): + """ Custom encoder for numpy data types """ + def default(self, obj): + if isinstance(obj, (np.int_, np.intc, np.intp, np.int8, + np.int16, np.int32, np.int64, np.uint8, + np.uint16, np.uint32, np.uint64)): + + return int(obj) + + elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)): + return float(obj) + + elif isinstance(obj, (np.complex_, np.complex64, np.complex128)): + return {'real': obj.real, 'imag': obj.imag} + + elif isinstance(obj, (np.ndarray,)): + return obj.tolist() + + elif isinstance(obj, (np.bool_)): + return bool(obj) + + elif isinstance(obj, (np.void)): + return None + + return json.JSONEncoder.default(self, obj) \ No newline at end of file From c15a4155f6225649688d33d1dbcf166ee05d2bcb Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 17 Jul 2024 15:31:15 +0200 Subject: [PATCH 4/9] fix(jsondump): add numpy encoder --- benchmark/src/benchmark.py | 4 +++- smac/callback/metadata_callback.py | 3 ++- smac/intensifier/abstract_intensifier.py | 3 ++- smac/main/smbo.py | 3 ++- smac/runhistory/runhistory.py | 4 +++- smac/utils/numpyencoder.py | 5 ++++- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/benchmark/src/benchmark.py b/benchmark/src/benchmark.py index 55cabd6d4..9ae74a5fb 100644 --- a/benchmark/src/benchmark.py +++ b/benchmark/src/benchmark.py @@ -17,6 +17,8 @@ from collections import defaultdict from pathlib import Path +from smac.utils.numpyencoder import NumpyEncoder + import pandas as pd from src.tasks import TASKS # noqa: E402 from src.utils.exceptions import NotSupportedError # noqa: E402 @@ -79,7 +81,7 @@ def _save_data(self) -> None: """Saves the internal data to the file.""" print("Saving data...") with open(str(RAW_FILENAME), "w") as f: - json.dump(self._data, f, indent=4) + json.dump(self._data, f, indent=4, cls=NumpyEncoder) def _fill_keys(self) -> None: """Fill data with keys based on computer name, tasks, and selected version.""" diff --git a/smac/callback/metadata_callback.py b/smac/callback/metadata_callback.py index 626de5dee..95a382b5f 100644 --- a/smac/callback/metadata_callback.py +++ b/smac/callback/metadata_callback.py @@ -7,6 +7,7 @@ import smac from smac.callback.callback import Callback from smac.main.smbo import SMBO +from smac.utils.numpyencoder import NumpyEncoder __copyright__ = "Copyright 2023, AutoML.org Freiburg-Hannover" __license__ = "3-clause BSD" @@ -31,4 +32,4 @@ def on_start(self, smbo: SMBO) -> None: path.mkdir(parents=True, exist_ok=True) with open(path / "metadata.json", "w") as fp: - json.dump(meta_dict, fp, indent=2) + json.dump(meta_dict, fp, indent=2, cls=NumpyEncoder) diff --git a/smac/intensifier/abstract_intensifier.py b/smac/intensifier/abstract_intensifier.py index cb537e9cc..5013fe113 100644 --- a/smac/intensifier/abstract_intensifier.py +++ b/smac/intensifier/abstract_intensifier.py @@ -27,6 +27,7 @@ from smac.utils.configspace import get_config_hash, print_config_changes from smac.utils.logging import get_logger from smac.utils.pareto_front import calculate_pareto_front, sort_by_crowding_distance +from smac.utils.numpyencoder import NumpyEncoder __copyright__ = "Copyright 2022, automl.org" __license__ = "3-clause BSD" @@ -666,7 +667,7 @@ def save(self, filename: str | Path) -> None: } with open(filename, "w") as fp: - json.dump(data, fp, indent=2) + json.dump(data, fp, indent=2, cls=NumpyEncoder) def load(self, filename: str | Path) -> None: """Loads the latest state of the intensifier including the incumbents and trajectory.""" diff --git a/smac/main/smbo.py b/smac/main/smbo.py index 6138e4d64..dd6bfd354 100644 --- a/smac/main/smbo.py +++ b/smac/main/smbo.py @@ -24,6 +24,7 @@ from smac.scenario import Scenario from smac.utils.data_structures import recursively_compare_dicts from smac.utils.logging import get_logger +from smac.utils.numpyencoder import NumpyEncoder __copyright__ = "Copyright 2022, automl.org" __license__ = "3-clause BSD" @@ -414,7 +415,7 @@ def save(self) -> None: # Save optimization data with open(str(path / "optimization.json"), "w") as file: - json.dump(data, file, indent=2) + json.dump(data, file, indent=2, cls=NumpyEncoder) # And save runhistory and intensifier self._runhistory.save(path / "runhistory.json") diff --git a/smac/runhistory/runhistory.py b/smac/runhistory/runhistory.py index c71384107..9200e678c 100644 --- a/smac/runhistory/runhistory.py +++ b/smac/runhistory/runhistory.py @@ -24,6 +24,7 @@ from smac.utils.configspace import get_config_hash from smac.utils.logging import get_logger from smac.utils.multi_objective import normalize_costs +from smac.utils.numpyencoder import NumpyEncoder __copyright__ = "Copyright 2022, automl.org" __license__ = "3-clause BSD" @@ -803,6 +804,7 @@ def save(self, filename: str | Path = "runhistory.json") -> None: }, fp, indent=2, + cls=NumpyEncoder ) def load(self, filename: str | Path, configspace: ConfigurationSpace) -> None: @@ -955,7 +957,7 @@ def _check_json_serializable( trial_value: TrialValue, ) -> None: try: - json.dumps(obj) + json.dumps(obj, cls=NumpyEncoder) except Exception as e: raise ValueError( "Cannot add %s: %s of type %s to runhistory because it raises an error during JSON encoding, " diff --git a/smac/utils/numpyencoder.py b/smac/utils/numpyencoder.py index ff05d9dbd..2cf624a9c 100644 --- a/smac/utils/numpyencoder.py +++ b/smac/utils/numpyencoder.py @@ -4,7 +4,10 @@ class NumpyEncoder(json.JSONEncoder): - """ Custom encoder for numpy data types """ + """ Custom encoder for numpy data types + + From https://stackoverflow.com/a/61903895 + """ def default(self, obj): if isinstance(obj, (np.int_, np.intc, np.intp, np.int8, np.int16, np.int32, np.int64, np.uint8, From f0ea0b7a9684405e7f8fb913e4fea041aeb487b1 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 17 Jul 2024 15:31:29 +0200 Subject: [PATCH 5/9] fix(ConfigSpace): parameter API --- smac/utils/configspace.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/smac/utils/configspace.py b/smac/utils/configspace.py index 8f281ed3e..8224f3ef9 100644 --- a/smac/utils/configspace.py +++ b/smac/utils/configspace.py @@ -101,22 +101,22 @@ def get_types( if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param._lower, param._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif isinstance(param, NormalIntegerHyperparameter): if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param.nfhp._lower, param.nfhp._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif isinstance(param, BetaFloatHyperparameter): if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param._lower, param._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif isinstance(param, BetaIntegerHyperparameter): if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param.bfhp._lower, param.bfhp._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif not isinstance( param, ( From 0a083646cab4f7d2635445882bf3c6b2001c1ebc Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 17 Jul 2024 15:31:38 +0200 Subject: [PATCH 6/9] Fix example (ConfigSpace API) --- examples/1_basics/6_priors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/1_basics/6_priors.py b/examples/1_basics/6_priors.py index 49b9aea26..691460c0b 100644 --- a/examples/1_basics/6_priors.py +++ b/examples/1_basics/6_priors.py @@ -95,8 +95,8 @@ def configspace(self) -> ConfigurationSpace: "learning_rate_init", lower=1e-5, upper=1.0, - mu=np.log(1e-3), - sigma=np.log(10), + mu=1e-3, # will be transformed to log space later + sigma=10, # will be transformed to log space later log=True, ) From b8a3b9ae12b310d33c2856bfb008469d4964545b Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 17 Jul 2024 15:39:01 +0200 Subject: [PATCH 7/9] Make format --- .../4_intensify_crossvalidation.py | 25 +++--- .../maximizer/local_and_random_search.py | 1 - .../initial_design/abstract_initial_design.py | 1 - smac/intensifier/abstract_intensifier.py | 2 +- .../hyperband_budget_calculation.ipynb | 73 +++++++++++++++++ smac/intensifier/successive_halving.py | 2 +- .../gaussian_process/kernels/rbf_kernel.py | 1 - .../gaussian_process/kernels/white_kernel.py | 1 - .../gaussian_process/mcmc_gaussian_process.py | 1 - smac/runhistory/runhistory.py | 2 +- smac/utils/numpyencoder.py | 35 +++++--- tests/conftest.py | 1 - tests/test_acquisition/test_maximizers.py | 3 +- .../test_abstract_intensifier.py | 27 ++----- tests/test_model/_test_gp_gpytorch.py | 1 - tests/test_model/test_gp.py | 3 - tests/test_model/test_rf.py | 1 - tests/test_runhistory/test_runhistory.py | 5 -- .../test_runhistory_encoder.py | 81 +++++-------------- 19 files changed, 141 insertions(+), 125 deletions(-) create mode 100644 smac/intensifier/hyperband_budget_calculation.ipynb diff --git a/examples/4_advanced_optimizer/4_intensify_crossvalidation.py b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py index 222d199f9..679253da1 100644 --- a/examples/4_advanced_optimizer/4_intensify_crossvalidation.py +++ b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py @@ -35,8 +35,8 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) # First we create our hyperparameters - C = Float("C", (2 ** - 5, 2 ** 15), default=1.0, log=True) - gamma = Float("gamma", (2 ** -15, 2 ** 3), default=1.0, log=True) + C = Float("C", (2**-5, 2**15), default=1.0, log=True) + gamma = Float("gamma", (2**-15, 2**3), default=1.0, log=True) # Add hyperparameters to our configspace cs.add([C, gamma]) @@ -45,7 +45,7 @@ def configspace(self) -> ConfigurationSpace: def train(self, config: Configuration, instance: str, seed: int = 0) -> float: """Creates a SVM based on a configuration and evaluate on the given fold of the digits dataset - + Parameters ---------- config: Configuration @@ -81,15 +81,14 @@ def train(self, config: Configuration, instance: str, seed: int = 0) -> float: scenario = Scenario( classifier.configspace, n_trials=50, # We want to run max 50 trials (combination of config and instances in the case of - # deterministic=True. In the case of deterministic=False, this would be the - # combination of instances, seeds and configs). The number of distinct configurations - # evaluated by SMAC will be lower than this number because some of the configurations - # will be executed on more than one instance (CV fold). + # deterministic=True. In the case of deterministic=False, this would be the + # combination of instances, seeds and configs). The number of distinct configurations + # evaluated by SMAC will be lower than this number because some of the configurations + # will be executed on more than one instance (CV fold). instances=[f"{i}" for i in range(N_FOLDS)], # Specify all instances by their name (as a string) - instance_features={f"{i}": [i] for i in range(N_FOLDS)}, # breaks SMAC + instance_features={f"{i}": [i] for i in range(N_FOLDS)}, # breaks SMAC deterministic=True # To simplify the problem we make SMAC believe that we have a deterministic - # optimization problem. - + # optimization problem. ) # We want to run the facade's default initial design, but we want to change the number @@ -102,12 +101,12 @@ def train(self, config: Configuration, instance: str, seed: int = 0) -> float: classifier.train, initial_design=initial_design, overwrite=True, # If the run exists, we overwrite it; alternatively, we can continue from last state - # The next line defines the intensifier, i.e., the module that governs the selection of + # The next line defines the intensifier, i.e., the module that governs the selection of # instance-seed pairs. Since we set deterministic to True above, it only governs the instance in # this example. Technically, it is not necessary to create the intensifier as a user, but it is # necessary to do so because we change the argument max_config_calls (the number of instance-seed pairs # per configuration to try) to the number of cross-validation folds, while the default would be 3. - intensifier=Intensifier(scenario=scenario, max_config_calls=N_FOLDS, seed=0) + intensifier=Intensifier(scenario=scenario, max_config_calls=N_FOLDS, seed=0), ) incumbent = smac.optimize() @@ -124,4 +123,4 @@ def train(self, config: Configuration, instance: str, seed: int = 0) -> float: # at more configurations than would have been possible with regular cross-validation, where the number # of configurations would be determined by the number of trials divided by the number of folds (50 / 10). runhistory = smac.runhistory - print(f"Number of evaluated configurations: {len(runhistory.config_ids)}") \ No newline at end of file + print(f"Number of evaluated configurations: {len(runhistory.config_ids)}") diff --git a/smac/acquisition/maximizer/local_and_random_search.py b/smac/acquisition/maximizer/local_and_random_search.py index d81af2f69..71c7f86c4 100644 --- a/smac/acquisition/maximizer/local_and_random_search.py +++ b/smac/acquisition/maximizer/local_and_random_search.py @@ -145,7 +145,6 @@ def _maximize( previous_configs: list[Configuration], n_points: int, ) -> list[tuple[float, Configuration]]: - if self._uniform_configspace is not None and self._prior_sampling_fraction is not None: # Get configurations sorted by acquisition function value next_configs_by_prior_random_search_sorted = self._prior_random_search._maximize( diff --git a/smac/initial_design/abstract_initial_design.py b/smac/initial_design/abstract_initial_design.py index 7e17c88c2..d561a6772 100644 --- a/smac/initial_design/abstract_initial_design.py +++ b/smac/initial_design/abstract_initial_design.py @@ -176,7 +176,6 @@ def _transform_continuous_designs( """ params = configspace.get_hyperparameters() for idx, param in enumerate(params): - if isinstance(param, IntegerHyperparameter): design[:, idx] = param._inverse_transform(param._transform(design[:, idx])) elif isinstance(param, NumericalHyperparameter): diff --git a/smac/intensifier/abstract_intensifier.py b/smac/intensifier/abstract_intensifier.py index 5013fe113..b7a5ae1ca 100644 --- a/smac/intensifier/abstract_intensifier.py +++ b/smac/intensifier/abstract_intensifier.py @@ -26,8 +26,8 @@ from smac.scenario import Scenario from smac.utils.configspace import get_config_hash, print_config_changes from smac.utils.logging import get_logger -from smac.utils.pareto_front import calculate_pareto_front, sort_by_crowding_distance from smac.utils.numpyencoder import NumpyEncoder +from smac.utils.pareto_front import calculate_pareto_front, sort_by_crowding_distance __copyright__ = "Copyright 2022, automl.org" __license__ = "3-clause BSD" diff --git a/smac/intensifier/hyperband_budget_calculation.ipynb b/smac/intensifier/hyperband_budget_calculation.ipynb new file mode 100644 index 000000000..53766dc75 --- /dev/null +++ b/smac/intensifier/hyperband_budget_calculation.ipynb @@ -0,0 +1,73 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "14 545679.0123456805\n", + "------------------------------ HYPERBAND IN MULTI-FIDELITY ------------------------------\n", + "total budget:\t\t 10000000\n", + "total number of trials:\t 3017\n", + "number of HB rounds:\t 14.80804387568556\n", + "\n", + "\t~~~~~~~~~~~~HYPERBAND ROUND\n", + "\teta:\t\t\t\t\t 3\n", + "\tmin budget per trial:\t\t\t 1000\n", + "\tmax budget per trial:\t\t\t 100000\n", + "\ttotal number of trials per HB round:\t 206\n", + "\tbudget used per HB round:\t\t 675308.6419753085\n", + "\tnumber of brackets:\t\t\t 4\n", + "\tbudgets per stage:\t\t\t {0: [1234.567901234568, 3703.7037037037035, 11111.111111111111, 33333.33333333333, 100000.0], 1: [3703.7037037037035, 11111.111111111111, 33333.33333333333, 100000.0], 2: [11111.111111111111, 33333.33333333333, 100000.0], 3: [33333.33333333333, 100000.0], 4: [100000.0]}\n", + "\tn configs per stage:\t\t\t {0: [81, 27, 9, 3, 1], 1: [34, 11, 3, 1], 2: [15, 5, 1], 3: [8, 2], 4: [5]}\n", + "-----------------------------------------------------------------------------------------\n", + "3017\n" + ] + } + ], + "source": [ + "from smac.intensifier.hyperband_utils import get_n_trials_for_hyperband_multifidelity\n", + "\n", + "# Specify total budget\n", + "total_budget = 10_000_000\n", + "\n", + "# Optionall specify eta, min_budget and max_budget to adjust HB\n", + "min_budget = 1000\n", + "max_budget = 100000 # R\n", + "eta = 3\n", + "\n", + "\n", + "n_trials = get_n_trials_for_hyperband_multifidelity(total_budget=total_budget, eta=eta, min_budget=min_budget, max_budget=max_budget, print_summary=True)\n", + "print(n_trials)\n", + "\n", + "\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/smac/intensifier/successive_halving.py b/smac/intensifier/successive_halving.py index 01b95c6d8..58960ca67 100644 --- a/smac/intensifier/successive_halving.py +++ b/smac/intensifier/successive_halving.py @@ -384,7 +384,7 @@ def __iter__(self) -> Iterator[TrialInfo]: # noqa: D102 logger.debug("Updating tracker:") # TODO: Process stages ascending or descending? - for (bracket, stage) in list(self._tracker.keys()): + for bracket, stage in list(self._tracker.keys()): pairs = self._tracker[(bracket, stage)].copy() for seed, configs in pairs: isb_keys = self._get_instance_seed_budget_keys_by_stage(bracket=bracket, stage=stage, seed=seed) diff --git a/smac/model/gaussian_process/kernels/rbf_kernel.py b/smac/model/gaussian_process/kernels/rbf_kernel.py index 5bf207658..13aaac49f 100644 --- a/smac/model/gaussian_process/kernels/rbf_kernel.py +++ b/smac/model/gaussian_process/kernels/rbf_kernel.py @@ -24,7 +24,6 @@ def __init__( has_conditions: bool = False, prior: AbstractPrior | None = None, ) -> None: - super().__init__( operate_on=operate_on, has_conditions=has_conditions, diff --git a/smac/model/gaussian_process/kernels/white_kernel.py b/smac/model/gaussian_process/kernels/white_kernel.py index a3fa4a61b..f8a7814a7 100644 --- a/smac/model/gaussian_process/kernels/white_kernel.py +++ b/smac/model/gaussian_process/kernels/white_kernel.py @@ -21,7 +21,6 @@ def __init__( has_conditions: bool = False, prior: AbstractPrior | None = None, ) -> None: - super().__init__( operate_on=operate_on, has_conditions=has_conditions, diff --git a/smac/model/gaussian_process/mcmc_gaussian_process.py b/smac/model/gaussian_process/mcmc_gaussian_process.py index 7c4ff4028..d6a098159 100644 --- a/smac/model/gaussian_process/mcmc_gaussian_process.py +++ b/smac/model/gaussian_process/mcmc_gaussian_process.py @@ -247,7 +247,6 @@ def _train( assert self._samples is not None for sample in self._samples: - if (sample < -50).any(): sample[sample < -50] = -50 if (sample > 50).any(): diff --git a/smac/runhistory/runhistory.py b/smac/runhistory/runhistory.py index 9200e678c..091e3f95b 100644 --- a/smac/runhistory/runhistory.py +++ b/smac/runhistory/runhistory.py @@ -804,7 +804,7 @@ def save(self, filename: str | Path = "runhistory.json") -> None: }, fp, indent=2, - cls=NumpyEncoder + cls=NumpyEncoder, ) def load(self, filename: str | Path, configspace: ConfigurationSpace) -> None: diff --git a/smac/utils/numpyencoder.py b/smac/utils/numpyencoder.py index 2cf624a9c..5c31285c3 100644 --- a/smac/utils/numpyencoder.py +++ b/smac/utils/numpyencoder.py @@ -1,25 +1,40 @@ from __future__ import annotations -import numpy as np + import json +import numpy as np + class NumpyEncoder(json.JSONEncoder): - """ Custom encoder for numpy data types - + """Custom encoder for numpy data types + From https://stackoverflow.com/a/61903895 """ - def default(self, obj): - if isinstance(obj, (np.int_, np.intc, np.intp, np.int8, - np.int16, np.int32, np.int64, np.uint8, - np.uint16, np.uint32, np.uint64)): + def default(self, obj): + if isinstance( + obj, + ( + np.int_, + np.intc, + np.intp, + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.uint16, + np.uint32, + np.uint64, + ), + ): return int(obj) elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)): return float(obj) elif isinstance(obj, (np.complex_, np.complex64, np.complex128)): - return {'real': obj.real, 'imag': obj.imag} + return {"real": obj.real, "imag": obj.imag} elif isinstance(obj, (np.ndarray,)): return obj.tolist() @@ -27,7 +42,7 @@ def default(self, obj): elif isinstance(obj, (np.bool_)): return bool(obj) - elif isinstance(obj, (np.void)): + elif isinstance(obj, (np.void)): return None - return json.JSONEncoder.default(self, obj) \ No newline at end of file + return json.JSONEncoder.default(self, obj) diff --git a/tests/conftest.py b/tests/conftest.py index 92a415162..5b26d6b70 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -143,7 +143,6 @@ def pytest_sessionfinish(session: Session, exitstatus: ExitCode) -> None: proc = psutil.Process() kill_signal = signal.SIGTERM for child in proc.children(recursive=True): - # https://stackoverflow.com/questions/57336095/access-verbosity-level-in-a-pytest-helper-function if session.config.getoption("verbose") > 0: print(child, child.cmdline()) diff --git a/tests/test_acquisition/test_maximizers.py b/tests/test_acquisition/test_maximizers.py index 945dfba7f..d7698e0a2 100644 --- a/tests/test_acquisition/test_maximizers.py +++ b/tests/test_acquisition/test_maximizers.py @@ -262,7 +262,6 @@ def predict_marginalized(self, X): return X, X class AcquisitionFunction: - model = Model() def __call__(self, X): @@ -333,7 +332,7 @@ def test_local_and_random_search(configspace, acquisition_function): values = rs._maximize(start_points, 100) config_origins = [] v_old = np.inf - for (v, config) in values: + for v, config in values: config_origins += [config.origin] if isinstance(v, np.ndarray): v = float(v[0]) diff --git a/tests/test_intensifier/test_abstract_intensifier.py b/tests/test_intensifier/test_abstract_intensifier.py index b8dc91a1c..6d878f915 100644 --- a/tests/test_intensifier/test_abstract_intensifier.py +++ b/tests/test_intensifier/test_abstract_intensifier.py @@ -110,7 +110,7 @@ def test_incumbent_selection_multi_objective(make_scenario, configspace_small, m def test_config_rejection_single_objective(configspace_small, make_scenario): - """ Tests whether configs are rejected properly if they are worse than the incumbent. """ + """Tests whether configs are rejected properly if they are worse than the incumbent.""" scenario = make_scenario(configspace_small, use_instances=False) runhistory = RunHistory() intensifier = Intensifier(scenario=scenario) @@ -118,36 +118,21 @@ def test_config_rejection_single_objective(configspace_small, make_scenario): configs = configspace_small.sample_configuration(3) - runhistory.add(config=configs[0], - cost=5, - time=0.0, - seed=0, - status=StatusType.SUCCESS, - force_update=True) + runhistory.add(config=configs[0], cost=5, time=0.0, seed=0, status=StatusType.SUCCESS, force_update=True) intensifier.update_incumbents(configs[0]) assert intensifier._rejected_config_ids == [] # add config that yielded better results, updating incumbent and sending prior incumbent to rejected - runhistory.add(config=configs[1], - cost=1, - time=0.0, - seed=0, - status=StatusType.SUCCESS, - force_update=True) + runhistory.add(config=configs[1], cost=1, time=0.0, seed=0, status=StatusType.SUCCESS, force_update=True) intensifier.update_incumbents(config=configs[1]) - + assert intensifier._rejected_config_ids == [1] # add config that is no better should thus go to rejected - runhistory.add(config=configs[2], - cost=1, - time=0.0, - seed=0, - status=StatusType.SUCCESS, - force_update=True) + runhistory.add(config=configs[2], cost=1, time=0.0, seed=0, status=StatusType.SUCCESS, force_update=True) intensifier.update_incumbents(config=configs[2]) - + assert intensifier._rejected_config_ids == [1, 3] diff --git a/tests/test_model/_test_gp_gpytorch.py b/tests/test_model/_test_gp_gpytorch.py index 95f3a59d8..d9e90f9b3 100644 --- a/tests/test_model/_test_gp_gpytorch.py +++ b/tests/test_model/_test_gp_gpytorch.py @@ -325,7 +325,6 @@ def test_sampling_shape(self): X = np.arange(-5, 5, 0.1).reshape((-1, 1)) X_test = np.arange(-5.05, 5.05, 0.1).reshape((-1, 1)) for shape in (None, (-1, 1)): - if shape is None: y = np.sin(X).flatten() else: diff --git a/tests/test_model/test_gp.py b/tests/test_model/test_gp.py index 0c72eb2fa..9e21925bc 100644 --- a/tests/test_model/test_gp.py +++ b/tests/test_model/test_gp.py @@ -240,7 +240,6 @@ def __call__(self, X, eval_gradient=True, clone_kernel=True): raise np.linalg.LinAlgError with patch.object(sklearn.gaussian_process.GaussianProcessRegressor, "log_marginal_likelihood", Dummy().__call__): - seed = 1 rs = np.random.RandomState(seed) X, Y, n_dims = get_cont_data(rs) @@ -265,7 +264,6 @@ def __call__(self, X, Y=None): dummy = Dummy() with patch.object(GaussianProcess, "predict", dummy.__call__): - seed = 1 rs = np.random.RandomState(seed) @@ -375,7 +373,6 @@ def test_sampling_shape(): X = np.arange(-5, 5, 0.1).reshape((-1, 1)) X_test = np.arange(-5.05, 5.05, 0.1).reshape((-1, 1)) for shape in (None, (-1, 1)): - if shape is None: y = np.sin(X).flatten() else: diff --git a/tests/test_model/test_rf.py b/tests/test_model/test_rf.py index e59b2ded1..c81549a16 100644 --- a/tests/test_model/test_rf.py +++ b/tests/test_model/test_rf.py @@ -127,7 +127,6 @@ def test_predict_marginalized(): def test_predict_marginalized_mocked(): - rs = np.random.RandomState(1) F = {} for i in range(10): diff --git a/tests/test_runhistory/test_runhistory.py b/tests/test_runhistory/test_runhistory.py index a1b7616ca..428ec54ad 100644 --- a/tests/test_runhistory/test_runhistory.py +++ b/tests/test_runhistory/test_runhistory.py @@ -75,7 +75,6 @@ def test_add_and_pickle(runhistory, config1): def test_illegal_input(runhistory): - with pytest.raises(TypeError, match="Configuration must not be None."): runhistory.add(config=None, cost=1.23, time=2.34, status=StatusType.SUCCESS) @@ -87,7 +86,6 @@ def test_illegal_input(runhistory): def test_add_multiple_times(runhistory, config1): - for i in range(5): runhistory.add( config=config1, @@ -294,7 +292,6 @@ def test_full_update2(runhistory, config1, config2): def test_incremental_update(runhistory, config1): - runhistory.add( config=config1, cost=10, @@ -319,7 +316,6 @@ def test_incremental_update(runhistory, config1): def test_multiple_budgets(runhistory, config1): - runhistory.add( config=config1, cost=10, @@ -382,7 +378,6 @@ def test_get_configs_per_budget(runhistory, config1, config2, config3): def test_json_origin(configspace_small, config1): - for i, origin in enumerate(["test_origin", None]): config1.origin = origin runhistory = RunHistory() diff --git a/tests/test_runhistory/test_runhistory_encoder.py b/tests/test_runhistory/test_runhistory_encoder.py index e3a8f5730..d9aaf0769 100644 --- a/tests/test_runhistory/test_runhistory_encoder.py +++ b/tests/test_runhistory/test_runhistory_encoder.py @@ -1,5 +1,7 @@ import numpy as np import pytest +from ConfigSpace import Configuration +from ConfigSpace.hyperparameters import CategoricalHyperparameter from smac.multi_objective.aggregation_strategy import MeanAggregationStrategy from smac.runhistory.encoder import ( @@ -13,9 +15,6 @@ from smac.runhistory.encoder.encoder import RunHistoryEncoder from smac.runner.abstract_runner import StatusType -from ConfigSpace import Configuration -from ConfigSpace.hyperparameters import CategoricalHyperparameter - @pytest.fixture def configs(configspace_small): @@ -42,9 +41,7 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): ) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory # TODO: Please replace with the more general solution once ConfigSpace 1.0 @@ -54,9 +51,7 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): # Categoricals are upperbounded by their size, rest of hyperparameters are # upperbounded by 1. upper_bounds = { - hp.name: (hp.get_size() - 1) - if isinstance(hp, CategoricalHyperparameter) - else 1.0 + hp.name: (hp.get_size() - 1) if isinstance(hp, CategoricalHyperparameter) else 1.0 for hp in configspace_small.get_hyperparameters() } # Need to ensure they match the order in the Configuration vectorized form @@ -73,49 +68,37 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): assert ((X1 <= upper) & (X1 >= lower)).all() # Log encoder - encoder = RunHistoryLogEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryLogEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryLogScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryLogScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryInverseScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryInverseScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistorySqrtScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistorySqrtScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryEIPSEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEIPSEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() @@ -160,9 +143,7 @@ def test_transform_conditionals(runhistory, make_scenario, configspace_large): status=StatusType.SUCCESS, ) - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() @@ -184,9 +165,7 @@ def test_multi_objective(runhistory, make_scenario, configspace_small, configs): # Multi objective algorithm must be set with pytest.raises(AssertionError): - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory _, Y = encoder.transform() @@ -242,9 +221,7 @@ def test_ignore(runhistory, make_scenario, configspace_small, configs): ) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X1, Y1 = encoder.transform() @@ -283,14 +260,10 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): budget=2, ) - runhistory.add( - config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2 - ) + runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform(budget_subset=[2]) assert Y.tolist() == [[99999999]] @@ -319,14 +292,10 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): budget=2, ) - runhistory.add( - config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2 - ) + runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform(budget_subset=[2]) assert Y.tolist() == [[99999999]] @@ -338,20 +307,12 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): def test_lower_budget_states(runhistory, make_scenario, configspace_small, configs): """Tests lower budgets based on budget subset and considered states.""" scenario = make_scenario(configspace_small) - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory - runhistory.add( - config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3 - ) - runhistory.add( - config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4 - ) - runhistory.add( - config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5 - ) + runhistory.add(config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3) + runhistory.add(config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4) + runhistory.add(config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5) # We request a higher budget but can't find it, so we expect an empty list X, Y = encoder.transform(budget_subset=[500]) From 475dbb23017cae2981e0e8802ed80757e8cbd3f6 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 17 Jul 2024 15:48:50 +0200 Subject: [PATCH 8/9] Fix pre-commit --- smac/utils/numpyencoder.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/smac/utils/numpyencoder.py b/smac/utils/numpyencoder.py index 5c31285c3..8bf67b773 100644 --- a/smac/utils/numpyencoder.py +++ b/smac/utils/numpyencoder.py @@ -3,6 +3,7 @@ import json import numpy as np +from typing import Any class NumpyEncoder(json.JSONEncoder): @@ -11,7 +12,19 @@ class NumpyEncoder(json.JSONEncoder): From https://stackoverflow.com/a/61903895 """ - def default(self, obj): + def default(self, obj: Any) -> Any: + """Handle numpy datatypes if present by converting to native python + + Parameters + ---------- + obj : Any + Object to serialize + + Returns + ------- + Any + Object in native python + """ if isinstance( obj, ( From d54087cb05b0a6d0361eae959882f172543a6f62 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 17 Jul 2024 15:49:30 +0200 Subject: [PATCH 9/9] isort --- smac/utils/numpyencoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/smac/utils/numpyencoder.py b/smac/utils/numpyencoder.py index 8bf67b773..c7b2b6c7b 100644 --- a/smac/utils/numpyencoder.py +++ b/smac/utils/numpyencoder.py @@ -1,9 +1,10 @@ from __future__ import annotations +from typing import Any + import json import numpy as np -from typing import Any class NumpyEncoder(json.JSONEncoder):