Optimizer Evaluation Tools

source/Mlos.Proto/OptimizerService.proto

@@ -18,10 +18,18 @@ service OptimizerService
     //
     rpc GetOptimizerConvergenceState(OptimizerHandle) returns (OptimizerConvergenceState);
 
+    // Returns the computed goodness of fit metrics.
+    //
+    rpc ComputeGoodnessOfFitMetrics(OptimizerHandle) returns (SimpleString);
+
     // Creates an optimizer with the specified configuration.
     //
     rpc CreateOptimizer(CreateOptimizerRequest) returns (OptimizerHandle);
-
+
+    // Check if optimizer has been trained.
+    //
+    rpc IsTrained(OptimizerHandle) returns (SimpleBoolean);
+
     // Request a suggestion.
     //
     rpc Suggest(SuggestRequest) returns (ConfigurationParameters);
@@ -208,6 +216,20 @@ message PredictResponse
     repeated SingleObjectivePrediction ObjectivePredictions = 1;
 };
 
+// A message containing a single boolean value.
+//
+message SimpleBoolean
+{
+    bool Value = 1;
+};
+
+// A message containing a single string value.
+//
+message SimpleString
+{
+    string Value = 1;
+}
+
 // A convenience message for optimizer functions that return nothing.
 //
 message Empty

source/Mlos.Python/mlos/Grpc/BayesianOptimizerProxy.py

@@ -11,11 +11,11 @@
 from mlos.Grpc import OptimizerService_pb2, OptimizerService_pb2_grpc
 from mlos.Logger import create_logger
 from mlos.Optimizers.OptimizerBase import OptimizerBase
+from mlos.Optimizers.RegressionModels.GoodnessOfFitMetrics import GoodnessOfFitMetrics
 from mlos.Optimizers.RegressionModels.Prediction import Prediction
 from mlos.Spaces import Point
 from mlos.Tracer import trace
 
-
 class BayesianOptimizerProxy(OptimizerBase):
     """ Client to remote BayesianOptimizer.
 
@@ -63,11 +63,21 @@ def __init__(
     def optimizer_handle(self):
         return OptimizerService_pb2.OptimizerHandle(Id=self.id)
 
+    @property
+    def trained(self):
+        response = self._optimizer_stub.IsTrained(self.optimizer_handle)
+        return response.Value
+
     @trace()
     def get_optimizer_convergence_state(self):
         optimizer_convergence_state_response = self._optimizer_stub.GetOptimizerConvergenceState(self.optimizer_handle)
         return deserialize_from_bytes_string(optimizer_convergence_state_response.SerializedOptimizerConvergenceState)
 
+    @trace()
+    def compute_surrogate_model_goodness_of_fit(self):
+        response = self._optimizer_stub.ComputeGoodnessOfFitMetrics(self.optimizer_handle)
+        return GoodnessOfFitMetrics.from_json(response.Value)
+
     @trace()
     def suggest(self, random=False, context=None):  # pylint: disable=unused-argument
         suggestion_request = OptimizerService_pb2.SuggestRequest(

source/Mlos.Python/mlos/Grpc/OptimizerMicroservice.py

@@ -14,7 +14,7 @@
 from mlos.global_values import serialize_to_bytes_string
 from mlos.Grpc import OptimizerService_pb2, OptimizerService_pb2_grpc
 from mlos.Grpc.OptimizerService_pb2 import Empty, OptimizerConvergenceState, OptimizerInfo, OptimizerHandle, OptimizerList, Observations, Features,\
-    ObjectiveValues
+    ObjectiveValues, SimpleBoolean, SimpleString
 from mlos.Optimizers.BayesianOptimizer import BayesianOptimizer, bayesian_optimizer_config_store
 from mlos.Optimizers.OptimizationProblem import OptimizationProblem
 from mlos.Optimizers.RegressionModels.Prediction import Prediction
@@ -118,6 +118,16 @@ def CreateOptimizer(self, request: OptimizerService_pb2.CreateOptimizerRequest,
         self.logger.info(f"Created optimizer {optimizer_id}.")
         return OptimizerService_pb2.OptimizerHandle(Id=optimizer_id)
 
+    def IsTrained(self, request, context): # pylint: disable=unused-argument
+        with self.exclusive_optimizer(optimizer_id=request.Id) as optimizer:
+            is_trained = optimizer.trained
+        return SimpleBoolean(Value=is_trained)
+
+    def ComputeGoodnessOfFitMetrics(self, request, context):
+        with self.exclusive_optimizer(optimizer_id=request.Id) as optimizer:
+            gof_metrics = optimizer.compute_surrogate_model_goodness_of_fit()
+        return SimpleString(Value=gof_metrics.to_json())
+
     def Suggest(self, request, context): # pylint: disable=unused-argument
         self.logger.info("Suggesting")