[enh]: validate for bayesian optimization algorithm settings

pkg/suggestion/v1beta1/skopt/service.py

@@ -14,13 +14,14 @@
 
 import logging
 
+import grpc
+
 from pkg.apis.manager.v1beta1.python import api_pb2
 from pkg.apis.manager.v1beta1.python import api_pb2_grpc
+from pkg.suggestion.v1beta1.internal.base_health_service import HealthServicer
 from pkg.suggestion.v1beta1.internal.search_space import HyperParameterSearchSpace
 from pkg.suggestion.v1beta1.internal.trial import Trial, Assignment
 from pkg.suggestion.v1beta1.skopt.base_service import BaseSkoptService
-from pkg.suggestion.v1beta1.internal.base_health_service import HealthServicer
-
 
 logger = logging.getLogger(__name__)
 
@@ -36,10 +37,8 @@ def GetSuggestions(self, request, context):
  """
  Main function to provide suggestion.
  """
- algorithm_name, config = OptimizerConfiguration.convertAlgorithmSpec(
+ algorithm_name, config = OptimizerConfiguration.convert_algorithm_spec(
  request.experiment.spec.algorithm)
- if algorithm_name != "bayesianoptimization":
- raise Exception("Failed to create the algorithm: {}".format(algorithm_name))
 
  if self.is_first_run:
  search_space = HyperParameterSearchSpace.convert(request.experiment)
@@ -58,6 +57,15 @@ def GetSuggestions(self, request, context):
  parameter_assignments=Assignment.generate(new_trials)
  )
 
+ def ValidateAlgorithmSettings(self, request, context):
+ is_valid, message = OptimizerConfiguration.validate_algorithm_spec(
+ request.experiment.spec.algorithm)
+ if not is_valid:
+ context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
+ context.set_details(message)
+ logger.error(message)
+ return api_pb2.ValidateAlgorithmSettingsReply()
+
 
 class OptimizerConfiguration(object):
  def __init__(self, base_estimator="GP",
@@ -72,7 +80,7 @@ def __init__(self, base_estimator="GP",
  self.random_state = random_state
 
  @staticmethod
- def convertAlgorithmSpec(algorithm_spec):
+ def convert_algorithm_spec(algorithm_spec):
  optimizer = OptimizerConfiguration()
  for s in algorithm_spec.algorithm_settings:
  if s.name == "base_estimator":
@@ -86,3 +94,39 @@ def convertAlgorithmSpec(algorithm_spec):
  elif s.name == "random_state":
  optimizer.random_state = int(s.value)
  return algorithm_spec.algorithm_name, optimizer
+
+ @classmethod
+ def validate_algorithm_spec(cls, algorithm_spec):
+ algo_name = algorithm_spec.algorithm_name
+
+ if algo_name == "bayesianoptimization":
+ return cls._validate_bayesianoptimization_setting(algorithm_spec.algorithm_settings)
+ else:
+ return False, "unknown algorithm name {}".format(algo_name)
+
+ @classmethod
+ def _validate_bayesianoptimization_setting(cls, algorithm_settings):
+ for s in algorithm_settings:
+ try:
+ if s.name == "base_estimator":
+ if s.value not in ["GP", "RF", "ET", "GBRT"]:
+ return False, "base_estimator {} is not supported in Bayesian optimization".format(s.value)
+ elif s.name == "n_initial_points":
+ if not (int(s.value) >= 0):
+ return False, "n_initial_points should be great or equal than zero"
+ elif s.name == "acq_func":
+ if s.value not in ["gp_hedge", "LCB", "EI", "PI", "EIps", "PIps"]:
+ return False, "acq_func {} is not supported in Bayesian optimization".format(s.value)
+ elif s.name == "acq_optimizer":
+ if s.value not in ["auto", "sampling", "lbfgs"]:
+ return False, "acq_optimizer {} is not supported in Bayesian optimization".format(s.value)
+ elif s.name == "random_state":
+ if not (int(s.value) >= 0):
+ return False, "random_state should be great or equal than zero"
+ else:
+ return False, "unknown setting {} for algorithm bayesianoptimization".format(s.name)
+ except Exception as e:
+ return False, "failed to validate {name}({value}): {exception}".format(name=s.name, value=s.value,
+ exception=e)
+
+ return True, ""

test/suggestion/v1beta1/test_skopt_service.py

@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import unittest
+
 import grpc
 import grpc_testing
-import unittest
 
 from pkg.apis.manager.v1beta1.python import api_pb2
-
 from pkg.suggestion.v1beta1.skopt.service import SkoptService
 
 
@@ -177,8 +177,8 @@ def test_get_suggestion(self):
 
  get_suggestion = self.test_server.invoke_unary_unary(
  method_descriptor=(api_pb2.DESCRIPTOR
-  .services_by_name['Suggestion']
-  .methods_by_name['GetSuggestions']),
+ .services_by_name['Suggestion']
+ .methods_by_name['GetSuggestions']),
  invocation_metadata={},
  request=request, timeout=1)
 
@@ -187,6 +187,125 @@ def test_get_suggestion(self):
  self.assertEqual(code, grpc.StatusCode.OK)
  self.assertEqual(2, len(response.parameter_assignments))
 
+ def test_validate_algorithm_settings(self):
+ experiment_spec = [None]
+
+ def call_validate():
+ experiment = api_pb2.Experiment(name="test", spec=experiment_spec[0])
+ request = api_pb2.ValidateAlgorithmSettingsRequest(experiment=experiment)
+
+ validate_algorithm_settings = self.test_server.invoke_unary_unary(
+ method_descriptor=(api_pb2.DESCRIPTOR
+ .services_by_name['Suggestion']
+ .methods_by_name['ValidateAlgorithmSettings']),
+ invocation_metadata={},
+ request=request, timeout=1)
+
+ return validate_algorithm_settings.termination()
+
+ # valid cases
+ algorithm_spec = api_pb2.AlgorithmSpec(
+ algorithm_name="bayesianoptimization",
+ algorithm_settings=[
+ api_pb2.AlgorithmSetting(
+ name="random_state",
+ value="10"
+ )
+ ],
+ )
+ experiment_spec[0] = api_pb2.ExperimentSpec(algorithm=algorithm_spec)
+ self.assertEqual(call_validate()[2], grpc.StatusCode.OK)
+
+ # invalid cases
+ # unknown algorithm name
+ experiment_spec[0] = api_pb2.ExperimentSpec(
+ algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown"))
+ _, _, code, details = call_validate()
+ self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+ self.assertEqual(details, 'unknown algorithm name unknown')
+
+ # unknown config name
+ experiment_spec[0] = api_pb2.ExperimentSpec(
+ algorithm=api_pb2.AlgorithmSpec(
+ algorithm_name="bayesianoptimization",
+ algorithm_settings=[
+ api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")]
+ ))
+ _, _, code, details = call_validate()
+ self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+ self.assertEqual(details, 'unknown setting unknown_conf for algorithm bayesianoptimization')
+
+ # unknown base_estimator
+ experiment_spec[0] = api_pb2.ExperimentSpec(
+ algorithm=api_pb2.AlgorithmSpec(
+ algorithm_name="bayesianoptimization",
+ algorithm_settings=[
+ api_pb2.AlgorithmSetting(name="base_estimator", value="unknown estimator")]
+ ))
+ _, _, code, details = call_validate()
+ wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+ self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+ self.assertEqual(details,
+ "{name} {value} is not supported in Bayesian optimization".format(
+ name=wrong_algorithm_setting.name,
+ value=wrong_algorithm_setting.value))
+
+ # wrong n_initial_points
+ experiment_spec[0] = api_pb2.ExperimentSpec(
+ algorithm=api_pb2.AlgorithmSpec(
+ algorithm_name="bayesianoptimization",
+ algorithm_settings=[
+ api_pb2.AlgorithmSetting(name="n_initial_points", value="-1")]
+ ))
+ _, _, code, details = call_validate()
+ wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+ self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+ self.assertEqual(details, "{name} should be great or equal than zero".format(name=wrong_algorithm_setting.name))
+
+ # unknown acq_func
+ experiment_spec[0] = api_pb2.ExperimentSpec(
+ algorithm=api_pb2.AlgorithmSpec(
+ algorithm_name="bayesianoptimization",
+ algorithm_settings=[
+ api_pb2.AlgorithmSetting(name="acq_func", value="unknown")]
+ ))
+ _, _, code, details = call_validate()
+ wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+ self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+ self.assertEqual(details,
+ "{name} {value} is not supported in Bayesian optimization".format(
+ name=wrong_algorithm_setting.name,
+ value=wrong_algorithm_setting.value
+ ))
+
+ # unknown acq_optimizer
+ experiment_spec[0] = api_pb2.ExperimentSpec(
+ algorithm=api_pb2.AlgorithmSpec(
+ algorithm_name="bayesianoptimization",
+ algorithm_settings=[
+ api_pb2.AlgorithmSetting(name="acq_optimizer", value="unknown")]
+ ))
+ _, _, code, details = call_validate()
+ wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+ self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+ self.assertEqual(details,
+ "{name} {value} is not supported in Bayesian optimization".format(
+ name=wrong_algorithm_setting.name,
+ value=wrong_algorithm_setting.value
+ ))
+
+ # wrong random_state
+ experiment_spec[0] = api_pb2.ExperimentSpec(
+ algorithm=api_pb2.AlgorithmSpec(
+ algorithm_name="bayesianoptimization",
+ algorithm_settings=[
+ api_pb2.AlgorithmSetting(name="random_state", value="-1")]
+ ))
+ _, _, code, details = call_validate()
+ wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+ self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+ self.assertEqual(details, "{name} should be great or equal than zero".format(name=wrong_algorithm_setting.name))
+
 
 if __name__ == '__main__':
  unittest.main()