Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Validate possible operations for Grid suggestion #1205

Merged
merged 5 commits into from
Aug 11, 2021
Merged

Validate possible operations for Grid suggestion #1205

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
73b6fd3
Create common function to test validate algorithm settings
andreyvelich Jun 3, 2020
6a22286
remove parentheses
andreyvelich Jun 4, 2020
9c89dcb
Use common util to test Suggestions
andreyvelich Aug 9, 2021
4400176
Fix API name
andreyvelich Aug 9, 2021
9a4f1e5
Fix indexing
andreyvelich Aug 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 27 additions & 3 deletions pkg/suggestion/v1beta1/chocolate/service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,15 @@
from pkg.apis.manager.v1beta1.python import api_pb2
from pkg.apis.manager.v1beta1.python import api_pb2_grpc

from pkg.suggestion.v1beta1.internal.constant import DOUBLE
from pkg.suggestion.v1beta1.internal.constant import INTEGER, DOUBLE, CATEGORICAL, DISCRETE
from pkg.suggestion.v1beta1.internal.search_space import HyperParameterSearchSpace
from pkg.suggestion.v1beta1.internal.trial import Trial, Assignment
from pkg.suggestion.v1beta1.chocolate.base_service import BaseChocolateService
from pkg.suggestion.v1beta1.internal.base_health_service import HealthServicer

import numpy as np
import itertools

logger = logging.getLogger(__name__)


Expand All @@ -38,11 +41,32 @@ def ValidateAlgorithmSettings(self, request, context):
if algorithm_name == "grid":
search_space = HyperParameterSearchSpace.convert(
request.experiment)
available_space = {}
for param in search_space.params:
if param.type == DOUBLE:
if param.type == INTEGER:
available_space[param.name] = range(int(param.min), int(param.max)+1, int(param.step))

elif param.type == DOUBLE:
if param.step == "" or param.step is None:
return self._set_validate_context_error(
context, "param {} step is nil".format(param.name))
context, "Param: {} step is nil".format(param.name))
double_list = np.arange(float(param.min), float(param.max)+float(param.step), float(param.step))
if double_list[-1] > float(param.max):
double_list = double_list[:-1]
available_space[param.name] = double_list

elif param.type == CATEGORICAL or param.type == DISCRETE:
available_space[param.name] = param.list

num_combinations = len(list(itertools.product(*available_space.values())))
max_trial_count = request.experiment.spec.max_trial_count

if max_trial_count > num_combinations:
return self._set_validate_context_error(
context, "Max Trial Count: {} > all possible search space combinations: {}".format(
max_trial_count, num_combinations)
)

return api_pb2.ValidateAlgorithmSettingsReply()

def GetSuggestions(self, request, context):
Expand Down
99 changes: 99 additions & 0 deletions test/suggestion/v1beta1/test_chocolate_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

from pkg.suggestion.v1beta1.chocolate.service import ChocolateService

import utils


class TestChocolate(unittest.TestCase):
def setUp(self):
Expand Down Expand Up @@ -181,6 +183,103 @@ def test_get_suggestion(self):
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(2, len(response.parameter_assignments))

def test_validate_algorithm_settings(self):
# Valid case.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="grid",
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(
name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["cat1", "cat2", "cat3"])
),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])
),
api_pb2.ParameterSpec(
name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="2.9", min="1", list=[], step="0.5")
)
]
),
max_trial_count=12,
parallel_trial_count=3,
)

_, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.OK)

# Invalid cases.
# Empty step.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="grid",
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="3", min="1", list=[])
)
]
),
)

_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'Param: param-1 step is nil')

# Max trial count > search space combinations.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="grid",
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="2", min="1", list=[]),
),
api_pb2.ParameterSpec(
name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["cat1", "cat2"])
),
api_pb2.ParameterSpec(
name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="2", min="1", list=[], step="0.5")
)
]
),
max_trial_count=15,
)

_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'Max Trial Count: 15 > all possible search space combinations: 12')


if __name__ == '__main__':
unittest.main()
155 changes: 85 additions & 70 deletions test/suggestion/v1beta1/test_hyperopt_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@
from pkg.apis.manager.v1beta1.python import api_pb2
from pkg.suggestion.v1beta1.hyperopt.service import HyperoptService

import utils


class TestHyperopt(unittest.TestCase):
def setUp(self):
servicers = {
api_pb2.DESCRIPTOR.services_by_name['Suggestion']: HyperoptService(
api_pb2.DESCRIPTOR.services_by_name["Suggestion"]: HyperoptService(
)
}

Expand Down Expand Up @@ -189,8 +191,8 @@ def test_get_suggestion(self):

get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(api_pb2.DESCRIPTOR
.services_by_name['Suggestion']
.methods_by_name['GetSuggestions']),
.services_by_name["Suggestion"]
.methods_by_name["GetSuggestions"]),
invocation_metadata={},
request=request, timeout=1)

Expand All @@ -200,103 +202,116 @@ def test_get_suggestion(self):
self.assertEqual(2, len(response.parameter_assignments))

def test_validate_algorithm_settings(self):
experiment_spec = [None]

def call_validate():
experiment = api_pb2.Experiment(name="test", spec=experiment_spec[0])
request = api_pb2.ValidateAlgorithmSettingsRequest(experiment=experiment)

validate_algorithm_settings = self.test_server.invoke_unary_unary(
method_descriptor=(api_pb2.DESCRIPTOR
.services_by_name['Suggestion']
.methods_by_name['ValidateAlgorithmSettings']),
invocation_metadata={},
request=request, timeout=1)
# Valid cases.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(
name="random_state",
value="10"
),
api_pb2.AlgorithmSetting(
name="gamma",
value="0.25"
),
api_pb2.AlgorithmSetting(
name="prior_weight",
value="1.0"
),
api_pb2.AlgorithmSetting(
name="n_EI_candidates",
value="24"
),
]
)
)

return validate_algorithm_settings.termination()
_, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.OK)

# valid cases
algorithm_spec = api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(
name="random_state",
value="10"
),
api_pb2.AlgorithmSetting(
name="gamma",
value="0.25"
),
api_pb2.AlgorithmSetting(
name="prior_weight",
value="1.0"
),
api_pb2.AlgorithmSetting(
name="n_EI_candidates",
value="24"
),
],
# Invalid cases.
# Unknown algorithm name.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="unknown"
)
)
experiment_spec[0] = api_pb2.ExperimentSpec(algorithm=algorithm_spec)
self.assertEqual(call_validate()[2], grpc.StatusCode.OK)

# invalid cases
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown"))
_, _, code, details = call_validate()
_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'unknown algorithm name unknown')
self.assertEqual(details, "unknown algorithm name unknown")

experiment_spec[0] = api_pb2.ExperimentSpec(
# Unknown algorithm setting name.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="random",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")]
))
_, _, code, details = call_validate()
api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")
]
)
)

_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'unknown setting unknown_conf for algorithm random')
self.assertEqual(details, "unknown setting unknown_conf for algorithm random")

experiment_spec[0] = api_pb2.ExperimentSpec(
# Invalid gamma value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="gamma", value="1.5")]
))
_, _, code, details = call_validate()
api_pb2.AlgorithmSetting(name="gamma", value="1.5")
]
)
)

_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'gamma should be in the range of (0, 1)')
self.assertEqual(details, "gamma should be in the range of (0, 1)")

experiment_spec[0] = api_pb2.ExperimentSpec(
# Invalid n_EI_candidates value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="n_EI_candidates", value="0")]
))
_, _, code, details = call_validate()
api_pb2.AlgorithmSetting(name="n_EI_candidates", value="0")
]
)
)

_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'n_EI_candidates should be great than zero')
self.assertEqual(details, "n_EI_candidates should be great than zero")

experiment_spec[0] = api_pb2.ExperimentSpec(
# Invalid random_state value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state", value="-1")]
))
_, _, code, details = call_validate()
api_pb2.AlgorithmSetting(name="random_state", value="-1")
]
)
)

_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'random_state should be great or equal than zero')
self.assertEqual(details, "random_state should be great or equal than zero")

experiment_spec[0] = api_pb2.ExperimentSpec(
# Invalid prior_weight value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="prior_weight", value="aaa")]
))
_, _, code, details = call_validate()
api_pb2.AlgorithmSetting(name="prior_weight", value="aaa")
]
)
)

_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertTrue(details.startswith('failed to validate prior_weight(aaa)'))
self.assertTrue(details.startswith("failed to validate prior_weight(aaa)"))


if __name__ == '__main__':
if __name__ == "__main__":
unittest.main()
Loading