Optimizer Evaluation Tools (#155)

Optimzier Evaluation Tools

The bulk of this PR lives in two files:
- OptimizerEvaluationReport.py
- OptimizerEvaluator.py
Their usage is presented in the OptimizerEvaluationTools.ipynb Notebook.

The goal of this work is to produce a framework to efficiently compare optimizer configurations across a variety of synthetic objective functions.

source/Mlos.Proto/OptimizerService.proto

@@ -18,10 +18,18 @@ service OptimizerService
  //
  rpc GetOptimizerConvergenceState(OptimizerHandle) returns (OptimizerConvergenceState);
 
+ // Returns the computed goodness of fit metrics.
+ //
+ rpc ComputeGoodnessOfFitMetrics(OptimizerHandle) returns (SimpleString);
+
  // Creates an optimizer with the specified configuration.
  //
  rpc CreateOptimizer(CreateOptimizerRequest) returns (OptimizerHandle);
-
+
+ // Check if optimizer has been trained.
+ //
+ rpc IsTrained(OptimizerHandle) returns (SimpleBoolean);
+
  // Request a suggestion.
  //
  rpc Suggest(SuggestRequest) returns (ConfigurationParameters);
@@ -208,6 +216,20 @@ message PredictResponse
  repeated SingleObjectivePrediction ObjectivePredictions = 1;
 };
 
+// A message containing a single boolean value.
+//
+message SimpleBoolean
+{
+ bool Value = 1;
+};
+
+// A message containing a single string value.
+//
+message SimpleString
+{
+ string Value = 1;
+}
+
 // A convenience message for optimizer functions that return nothing.
 //
 message Empty

source/Mlos.Python/mlos/Grpc/BayesianOptimizerProxy.py

@@ -11,11 +11,11 @@
 from mlos.Grpc import OptimizerService_pb2, OptimizerService_pb2_grpc
 from mlos.Logger import create_logger
 from mlos.Optimizers.OptimizerBase import OptimizerBase
+from mlos.Optimizers.RegressionModels.GoodnessOfFitMetrics import GoodnessOfFitMetrics
 from mlos.Optimizers.RegressionModels.Prediction import Prediction
 from mlos.Spaces import Point
 from mlos.Tracer import trace
 
-
 class BayesianOptimizerProxy(OptimizerBase):
  """ Client to remote BayesianOptimizer.
 
@@ -63,11 +63,21 @@ def __init__(
  def optimizer_handle(self):
  return OptimizerService_pb2.OptimizerHandle(Id=self.id)
 
+ @property
+ def trained(self):
+ response = self._optimizer_stub.IsTrained(self.optimizer_handle)
+ return response.Value
+
  @trace()
  def get_optimizer_convergence_state(self):
  optimizer_convergence_state_response = self._optimizer_stub.GetOptimizerConvergenceState(self.optimizer_handle)
  return deserialize_from_bytes_string(optimizer_convergence_state_response.SerializedOptimizerConvergenceState)
 
+ @trace()
+ def compute_surrogate_model_goodness_of_fit(self):
+ response = self._optimizer_stub.ComputeGoodnessOfFitMetrics(self.optimizer_handle)
+ return GoodnessOfFitMetrics.from_json(response.Value)
+
  @trace()
  def suggest(self, random=False, context=None): # pylint: disable=unused-argument
  suggestion_request = OptimizerService_pb2.SuggestRequest(

source/Mlos.Python/mlos/Grpc/OptimizerMicroservice.py

@@ -14,7 +14,7 @@
 from mlos.global_values import serialize_to_bytes_string
 from mlos.Grpc import OptimizerService_pb2, OptimizerService_pb2_grpc
 from mlos.Grpc.OptimizerService_pb2 import Empty, OptimizerConvergenceState, OptimizerInfo, OptimizerHandle, OptimizerList, Observations, Features,\
- ObjectiveValues
+ ObjectiveValues, SimpleBoolean, SimpleString
 from mlos.Optimizers.BayesianOptimizer import BayesianOptimizer, bayesian_optimizer_config_store
 from mlos.Optimizers.OptimizationProblem import OptimizationProblem
 from mlos.Optimizers.RegressionModels.Prediction import Prediction
@@ -118,6 +118,16 @@ def CreateOptimizer(self, request: OptimizerService_pb2.CreateOptimizerRequest,
  self.logger.info(f"Created optimizer {optimizer_id}.")
  return OptimizerService_pb2.OptimizerHandle(Id=optimizer_id)
 
+ def IsTrained(self, request, context): # pylint: disable=unused-argument
+ with self.exclusive_optimizer(optimizer_id=request.Id) as optimizer:
+ is_trained = optimizer.trained
+ return SimpleBoolean(Value=is_trained)
+
+ def ComputeGoodnessOfFitMetrics(self, request, context):
+ with self.exclusive_optimizer(optimizer_id=request.Id) as optimizer:
+ gof_metrics = optimizer.compute_surrogate_model_goodness_of_fit()
+ return SimpleString(Value=gof_metrics.to_json())
+
  def Suggest(self, request, context): # pylint: disable=unused-argument
  self.logger.info("Suggesting")