From ae2672e1335bfc2add5e32229d551852df9e5f82 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Sat, 31 Jul 2021 13:27:06 +0900 Subject: [PATCH 1/2] [enh]: validate for skopt algorithm settings --- pkg/suggestion/v1beta1/skopt/service.py | 52 +++++++- test/suggestion/v1beta1/test_skopt_service.py | 119 +++++++++++++++++- 2 files changed, 163 insertions(+), 8 deletions(-) diff --git a/pkg/suggestion/v1beta1/skopt/service.py b/pkg/suggestion/v1beta1/skopt/service.py index 4f153b33b1b..8b468f5a13a 100644 --- a/pkg/suggestion/v1beta1/skopt/service.py +++ b/pkg/suggestion/v1beta1/skopt/service.py @@ -14,6 +14,8 @@ import logging +import grpc + from pkg.apis.manager.v1beta1.python import api_pb2 from pkg.apis.manager.v1beta1.python import api_pb2_grpc from pkg.suggestion.v1beta1.internal.search_space import HyperParameterSearchSpace @@ -38,8 +40,6 @@ def GetSuggestions(self, request, context): """ algorithm_name, config = OptimizerConfiguration.convertAlgorithmSpec( request.experiment.spec.algorithm) - if algorithm_name != "bayesianoptimization": - raise Exception("Failed to create the algorithm: {}".format(algorithm_name)) if self.is_first_run: search_space = HyperParameterSearchSpace.convert(request.experiment) @@ -58,6 +58,15 @@ def GetSuggestions(self, request, context): parameter_assignments=Assignment.generate(new_trials) ) + def ValidateAlgorithmSettings(self, request, context): + is_valid, message = OptimizerConfiguration.validate_algorithm_spec( + request.experiment.spec.algorithm) + if not is_valid: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(message) + logger.error(message) + return api_pb2.ValidateAlgorithmSettingsReply() + class OptimizerConfiguration(object): def __init__(self, base_estimator="GP", @@ -71,8 +80,8 @@ def __init__(self, base_estimator="GP", self.acq_optimizer = acq_optimizer self.random_state = random_state - @staticmethod - def convertAlgorithmSpec(algorithm_spec): + @classmethod + def convertAlgorithmSpec(cls, algorithm_spec): optimizer = OptimizerConfiguration() for s in algorithm_spec.algorithm_settings: if s.name == "base_estimator": @@ -86,3 +95,38 @@ def convertAlgorithmSpec(algorithm_spec): elif s.name == "random_state": optimizer.random_state = int(s.value) return algorithm_spec.algorithm_name, optimizer + + @classmethod + def validate_algorithm_spec(cls, algorithm_spec): + algo_name = algorithm_spec.algorithm_name + + if algo_name == "bayesianoptimization": + return cls._validate_bayesianoptimization_setting(algorithm_spec.algorithm_settings) + else: + return False, "unknown algorithm name {}".format(algo_name) + + @classmethod + def _validate_bayesianoptimization_setting(cls, algorithm_settings): + for s in algorithm_settings: + try: + if s.name == "base_estimator": + if s.value not in ["GP", "RF", "ET", "GBRT"]: + return False, f"base_estimator {s.value} is not supported in katib" + elif s.name == "n_initial_points": + if not (int(s.value) >= 0): + return False, "n_initial_points should be great or equal than zero" + elif s.name == "acq_func": + if s.value not in ["gp_hedge", "LCB", "EI", "PI", "EIps", "PIps"]: + return False, f"acq_func {s.value} is not supported in katib" + elif s.name == "acq_optimizer": + if s.value not in ["auto", "sampling", "lbfgs"]: + return False, f"acq_optimizer {s.value} is not supported in katib" + elif s.name == "random_state": + if not (int(s.value) >= 0): + return False, "random_state should be great or equal than zero" + else: + return False, f"unknown setting {s.name} for algorithm bayesianoptimization" + except Exception as e: + return False, f"failed to validate {s.name}({s.value}): {e}" + + return True, "" diff --git a/test/suggestion/v1beta1/test_skopt_service.py b/test/suggestion/v1beta1/test_skopt_service.py index 1bf623a8079..16d1cd17bde 100644 --- a/test/suggestion/v1beta1/test_skopt_service.py +++ b/test/suggestion/v1beta1/test_skopt_service.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import unittest + import grpc import grpc_testing -import unittest from pkg.apis.manager.v1beta1.python import api_pb2 - from pkg.suggestion.v1beta1.skopt.service import SkoptService @@ -177,8 +177,8 @@ def test_get_suggestion(self): get_suggestion = self.test_server.invoke_unary_unary( method_descriptor=(api_pb2.DESCRIPTOR - .services_by_name['Suggestion'] - .methods_by_name['GetSuggestions']), + .services_by_name['Suggestion'] + .methods_by_name['GetSuggestions']), invocation_metadata={}, request=request, timeout=1) @@ -187,6 +187,117 @@ def test_get_suggestion(self): self.assertEqual(code, grpc.StatusCode.OK) self.assertEqual(2, len(response.parameter_assignments)) + def test_validate_algorithm_settings(self): + experiment_spec = [None] + + def call_validate(): + experiment = api_pb2.Experiment(name="test", spec=experiment_spec[0]) + request = api_pb2.ValidateAlgorithmSettingsRequest(experiment=experiment) + + validate_algorithm_settings = self.test_server.invoke_unary_unary( + method_descriptor=(api_pb2.DESCRIPTOR + .services_by_name['Suggestion'] + .methods_by_name['ValidateAlgorithmSettings']), + invocation_metadata={}, + request=request, timeout=1) + + return validate_algorithm_settings.termination() + + # valid cases + algorithm_spec = api_pb2.AlgorithmSpec( + algorithm_name="bayesianoptimization", + algorithm_settings=[ + api_pb2.AlgorithmSetting( + name="random_state", + value="10" + ) + ], + ) + experiment_spec[0] = api_pb2.ExperimentSpec(algorithm=algorithm_spec) + self.assertEqual(call_validate()[2], grpc.StatusCode.OK) + + # invalid cases + # unknown algorithm name + experiment_spec[0] = api_pb2.ExperimentSpec( + algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown")) + _, _, code, details = call_validate() + self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) + self.assertEqual(details, 'unknown algorithm name unknown') + + # unknown config name + experiment_spec[0] = api_pb2.ExperimentSpec( + algorithm=api_pb2.AlgorithmSpec( + algorithm_name="bayesianoptimization", + algorithm_settings=[ + api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")] + )) + _, _, code, details = call_validate() + self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) + self.assertEqual(details, 'unknown setting unknown_conf for algorithm bayesianoptimization') + + # unknown base_estimator + experiment_spec[0] = api_pb2.ExperimentSpec( + algorithm=api_pb2.AlgorithmSpec( + algorithm_name="bayesianoptimization", + algorithm_settings=[ + api_pb2.AlgorithmSetting(name="base_estimator", value="unknown estimator")] + )) + _, _, code, details = call_validate() + wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0] + self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) + self.assertEqual(details, + f'{wrong_algorithm_setting.name} {wrong_algorithm_setting.value} is not supported in katib') + + # wrong n_initial_points + experiment_spec[0] = api_pb2.ExperimentSpec( + algorithm=api_pb2.AlgorithmSpec( + algorithm_name="bayesianoptimization", + algorithm_settings=[ + api_pb2.AlgorithmSetting(name="n_initial_points", value="-1")] + )) + _, _, code, details = call_validate() + wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0] + self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) + self.assertEqual(details, f'{wrong_algorithm_setting.name} should be great or equal than zero') + + # unknown acq_func + experiment_spec[0] = api_pb2.ExperimentSpec( + algorithm=api_pb2.AlgorithmSpec( + algorithm_name="bayesianoptimization", + algorithm_settings=[ + api_pb2.AlgorithmSetting(name="acq_func", value="unknown")] + )) + _, _, code, details = call_validate() + wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0] + self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) + self.assertEqual(details, + f'{wrong_algorithm_setting.name} {wrong_algorithm_setting.value} is not supported in katib') + + # unknown acq_optimizer + experiment_spec[0] = api_pb2.ExperimentSpec( + algorithm=api_pb2.AlgorithmSpec( + algorithm_name="bayesianoptimization", + algorithm_settings=[ + api_pb2.AlgorithmSetting(name="acq_optimizer", value="unknown")] + )) + _, _, code, details = call_validate() + wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0] + self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) + self.assertEqual(details, + f'{wrong_algorithm_setting.name} {wrong_algorithm_setting.value} is not supported in katib') + + # wrong random_state + experiment_spec[0] = api_pb2.ExperimentSpec( + algorithm=api_pb2.AlgorithmSpec( + algorithm_name="bayesianoptimization", + algorithm_settings=[ + api_pb2.AlgorithmSetting(name="random_state", value="-1")] + )) + _, _, code, details = call_validate() + wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0] + self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) + self.assertEqual(details, f'{wrong_algorithm_setting.name} should be great or equal than zero') + if __name__ == '__main__': unittest.main() From ddaf191c8dd299d1a9c7ffae6cf8d267ddc9a5e9 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Tue, 3 Aug 2021 10:10:27 +0900 Subject: [PATCH 2/2] [style]: refactor with reviews - use staticmethod rather than classmethod - change convertAlgorithmSpec method name to a snake_case - use .format() rather than f-string Signed-off-by: Jaeyeon Kim --- pkg/suggestion/v1beta1/skopt/service.py | 20 +++++++++---------- test/suggestion/v1beta1/test_skopt_service.py | 18 ++++++++++++----- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/pkg/suggestion/v1beta1/skopt/service.py b/pkg/suggestion/v1beta1/skopt/service.py index 8b468f5a13a..781391be67c 100644 --- a/pkg/suggestion/v1beta1/skopt/service.py +++ b/pkg/suggestion/v1beta1/skopt/service.py @@ -18,11 +18,10 @@ from pkg.apis.manager.v1beta1.python import api_pb2 from pkg.apis.manager.v1beta1.python import api_pb2_grpc +from pkg.suggestion.v1beta1.internal.base_health_service import HealthServicer from pkg.suggestion.v1beta1.internal.search_space import HyperParameterSearchSpace from pkg.suggestion.v1beta1.internal.trial import Trial, Assignment from pkg.suggestion.v1beta1.skopt.base_service import BaseSkoptService -from pkg.suggestion.v1beta1.internal.base_health_service import HealthServicer - logger = logging.getLogger(__name__) @@ -38,7 +37,7 @@ def GetSuggestions(self, request, context): """ Main function to provide suggestion. """ - algorithm_name, config = OptimizerConfiguration.convertAlgorithmSpec( + algorithm_name, config = OptimizerConfiguration.convert_algorithm_spec( request.experiment.spec.algorithm) if self.is_first_run: @@ -80,8 +79,8 @@ def __init__(self, base_estimator="GP", self.acq_optimizer = acq_optimizer self.random_state = random_state - @classmethod - def convertAlgorithmSpec(cls, algorithm_spec): + @staticmethod + def convert_algorithm_spec(algorithm_spec): optimizer = OptimizerConfiguration() for s in algorithm_spec.algorithm_settings: if s.name == "base_estimator": @@ -111,22 +110,23 @@ def _validate_bayesianoptimization_setting(cls, algorithm_settings): try: if s.name == "base_estimator": if s.value not in ["GP", "RF", "ET", "GBRT"]: - return False, f"base_estimator {s.value} is not supported in katib" + return False, "base_estimator {} is not supported in Bayesian optimization".format(s.value) elif s.name == "n_initial_points": if not (int(s.value) >= 0): return False, "n_initial_points should be great or equal than zero" elif s.name == "acq_func": if s.value not in ["gp_hedge", "LCB", "EI", "PI", "EIps", "PIps"]: - return False, f"acq_func {s.value} is not supported in katib" + return False, "acq_func {} is not supported in Bayesian optimization".format(s.value) elif s.name == "acq_optimizer": if s.value not in ["auto", "sampling", "lbfgs"]: - return False, f"acq_optimizer {s.value} is not supported in katib" + return False, "acq_optimizer {} is not supported in Bayesian optimization".format(s.value) elif s.name == "random_state": if not (int(s.value) >= 0): return False, "random_state should be great or equal than zero" else: - return False, f"unknown setting {s.name} for algorithm bayesianoptimization" + return False, "unknown setting {} for algorithm bayesianoptimization".format(s.name) except Exception as e: - return False, f"failed to validate {s.name}({s.value}): {e}" + return False, "failed to validate {name}({value}): {exception}".format(name=s.name, value=s.value, + exception=e) return True, "" diff --git a/test/suggestion/v1beta1/test_skopt_service.py b/test/suggestion/v1beta1/test_skopt_service.py index 16d1cd17bde..771b548c6ab 100644 --- a/test/suggestion/v1beta1/test_skopt_service.py +++ b/test/suggestion/v1beta1/test_skopt_service.py @@ -246,7 +246,9 @@ def call_validate(): wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0] self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) self.assertEqual(details, - f'{wrong_algorithm_setting.name} {wrong_algorithm_setting.value} is not supported in katib') + "{name} {value} is not supported in Bayesian optimization".format( + name=wrong_algorithm_setting.name, + value=wrong_algorithm_setting.value)) # wrong n_initial_points experiment_spec[0] = api_pb2.ExperimentSpec( @@ -258,7 +260,7 @@ def call_validate(): _, _, code, details = call_validate() wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0] self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) - self.assertEqual(details, f'{wrong_algorithm_setting.name} should be great or equal than zero') + self.assertEqual(details, "{name} should be great or equal than zero".format(name=wrong_algorithm_setting.name)) # unknown acq_func experiment_spec[0] = api_pb2.ExperimentSpec( @@ -271,7 +273,10 @@ def call_validate(): wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0] self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) self.assertEqual(details, - f'{wrong_algorithm_setting.name} {wrong_algorithm_setting.value} is not supported in katib') + "{name} {value} is not supported in Bayesian optimization".format( + name=wrong_algorithm_setting.name, + value=wrong_algorithm_setting.value + )) # unknown acq_optimizer experiment_spec[0] = api_pb2.ExperimentSpec( @@ -284,7 +289,10 @@ def call_validate(): wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0] self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) self.assertEqual(details, - f'{wrong_algorithm_setting.name} {wrong_algorithm_setting.value} is not supported in katib') + "{name} {value} is not supported in Bayesian optimization".format( + name=wrong_algorithm_setting.name, + value=wrong_algorithm_setting.value + )) # wrong random_state experiment_spec[0] = api_pb2.ExperimentSpec( @@ -296,7 +304,7 @@ def call_validate(): _, _, code, details = call_validate() wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0] self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) - self.assertEqual(details, f'{wrong_algorithm_setting.name} should be great or equal than zero') + self.assertEqual(details, "{name} should be great or equal than zero".format(name=wrong_algorithm_setting.name)) if __name__ == '__main__':